2024-12-05 11:56:04,627 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-05 11:56:04,642 main DEBUG Took 0.012998 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 11:56:04,643 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 11:56:04,643 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 11:56:04,644 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 11:56:04,645 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,654 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 11:56:04,668 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,670 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,670 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,671 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,671 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,672 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,673 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,673 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,674 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,674 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,675 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,675 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,676 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,676 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,676 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,676 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,677 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,677 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,677 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,678 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,678 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,678 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,679 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,679 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:56:04,679 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,679 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 11:56:04,681 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:56:04,682 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 11:56:04,684 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 11:56:04,684 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 11:56:04,685 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 11:56:04,685 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 11:56:04,693 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 11:56:04,696 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 11:56:04,697 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 11:56:04,698 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 11:56:04,698 main DEBUG createAppenders(={Console}) 2024-12-05 11:56:04,699 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-05 11:56:04,699 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-05 11:56:04,699 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-05 11:56:04,700 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 11:56:04,700 main DEBUG OutputStream closed 2024-12-05 11:56:04,700 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 11:56:04,700 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 11:56:04,701 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-05 11:56:04,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 11:56:04,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 11:56:04,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 11:56:04,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 11:56:04,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 11:56:04,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 11:56:04,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 11:56:04,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 11:56:04,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 11:56:04,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 11:56:04,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 11:56:04,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 11:56:04,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 11:56:04,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 11:56:04,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 11:56:04,792 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 11:56:04,792 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 11:56:04,793 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 11:56:04,795 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 11:56:04,795 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-05 11:56:04,796 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 11:56:04,797 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-05T11:56:05,003 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51 2024-12-05 11:56:05,005 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 11:56:05,006 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T11:56:05,014 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-12-05T11:56:05,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T11:56:05,046 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/cluster_f9008675-76bd-e1e2-e855-619669955c54, deleteOnExit=true 2024-12-05T11:56:05,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-05T11:56:05,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/test.cache.data in system properties and HBase conf 2024-12-05T11:56:05,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T11:56:05,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/hadoop.log.dir in system properties and HBase conf 2024-12-05T11:56:05,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T11:56:05,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T11:56:05,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T11:56:05,133 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T11:56:05,219 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T11:56:05,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T11:56:05,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T11:56:05,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T11:56:05,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T11:56:05,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T11:56:05,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T11:56:05,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T11:56:05,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T11:56:05,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T11:56:05,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/nfs.dump.dir in system properties and HBase conf 2024-12-05T11:56:05,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/java.io.tmpdir in system properties and HBase conf 2024-12-05T11:56:05,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T11:56:05,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T11:56:05,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T11:56:06,109 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T11:56:06,181 INFO [Time-limited test {}] log.Log(170): Logging initialized @2145ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T11:56:06,244 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T11:56:06,299 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T11:56:06,320 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T11:56:06,321 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T11:56:06,322 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T11:56:06,334 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T11:56:06,337 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/hadoop.log.dir/,AVAILABLE} 2024-12-05T11:56:06,337 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T11:56:06,504 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/java.io.tmpdir/jetty-localhost-46221-hadoop-hdfs-3_4_1-tests_jar-_-any-5827696058790551508/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T11:56:06,510 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:46221} 2024-12-05T11:56:06,510 INFO [Time-limited test {}] server.Server(415): Started @2475ms 2024-12-05T11:56:07,018 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T11:56:07,025 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T11:56:07,026 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T11:56:07,026 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T11:56:07,026 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T11:56:07,027 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/hadoop.log.dir/,AVAILABLE} 2024-12-05T11:56:07,028 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T11:56:07,123 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/java.io.tmpdir/jetty-localhost-43075-hadoop-hdfs-3_4_1-tests_jar-_-any-1573744448516303361/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T11:56:07,124 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:43075} 2024-12-05T11:56:07,124 INFO [Time-limited test {}] server.Server(415): Started @3089ms 2024-12-05T11:56:07,170 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T11:56:07,874 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/cluster_f9008675-76bd-e1e2-e855-619669955c54/data/data2/current/BP-897299292-172.17.0.2-1733399765732/current, will proceed with Du for space computation calculation, 2024-12-05T11:56:07,874 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/cluster_f9008675-76bd-e1e2-e855-619669955c54/data/data1/current/BP-897299292-172.17.0.2-1733399765732/current, will proceed with Du for space computation calculation, 2024-12-05T11:56:07,902 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T11:56:07,944 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc51fa883c2a6e044 with lease ID 0x2ab750effe24617e: Processing first storage report for DS-ee3e7562-f485-4c73-a36c-b29c62ebc773 from datanode DatanodeRegistration(127.0.0.1:36633, datanodeUuid=cf0fba41-3359-4e83-a698-6fd216eccf92, infoPort=46259, infoSecurePort=0, ipcPort=44867, storageInfo=lv=-57;cid=testClusterID;nsid=2137118968;c=1733399765732) 2024-12-05T11:56:07,945 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc51fa883c2a6e044 with lease ID 0x2ab750effe24617e: from storage DS-ee3e7562-f485-4c73-a36c-b29c62ebc773 node DatanodeRegistration(127.0.0.1:36633, datanodeUuid=cf0fba41-3359-4e83-a698-6fd216eccf92, infoPort=46259, infoSecurePort=0, ipcPort=44867, storageInfo=lv=-57;cid=testClusterID;nsid=2137118968;c=1733399765732), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T11:56:07,945 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc51fa883c2a6e044 with lease ID 0x2ab750effe24617e: Processing first storage report for DS-9fc742de-20b6-4c9f-88eb-7054d58f3705 from datanode DatanodeRegistration(127.0.0.1:36633, datanodeUuid=cf0fba41-3359-4e83-a698-6fd216eccf92, infoPort=46259, infoSecurePort=0, ipcPort=44867, storageInfo=lv=-57;cid=testClusterID;nsid=2137118968;c=1733399765732) 2024-12-05T11:56:07,945 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc51fa883c2a6e044 with lease ID 0x2ab750effe24617e: from storage DS-9fc742de-20b6-4c9f-88eb-7054d58f3705 node DatanodeRegistration(127.0.0.1:36633, datanodeUuid=cf0fba41-3359-4e83-a698-6fd216eccf92, infoPort=46259, infoSecurePort=0, ipcPort=44867, storageInfo=lv=-57;cid=testClusterID;nsid=2137118968;c=1733399765732), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T11:56:08,038 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51 2024-12-05T11:56:08,098 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/cluster_f9008675-76bd-e1e2-e855-619669955c54/zookeeper_0, clientPort=58051, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/cluster_f9008675-76bd-e1e2-e855-619669955c54/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/cluster_f9008675-76bd-e1e2-e855-619669955c54/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T11:56:08,108 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58051 2024-12-05T11:56:08,116 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:56:08,118 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:56:08,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741825_1001 (size=7) 2024-12-05T11:56:08,724 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c with version=8 2024-12-05T11:56:08,725 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/hbase-staging 2024-12-05T11:56:08,808 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T11:56:09,038 INFO [Time-limited test {}] client.ConnectionUtils(128): master/80666a11a09f:0 server-side Connection retries=45 2024-12-05T11:56:09,046 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T11:56:09,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T11:56:09,051 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T11:56:09,051 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T11:56:09,051 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T11:56:09,163 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T11:56:09,214 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T11:56:09,222 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T11:56:09,226 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T11:56:09,248 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 104288 (auto-detected) 2024-12-05T11:56:09,249 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-05T11:56:09,266 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33741 2024-12-05T11:56:09,287 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33741 connecting to ZooKeeper ensemble=127.0.0.1:58051 2024-12-05T11:56:09,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:337410x0, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T11:56:09,393 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33741-0x101a6a30e880000 connected 2024-12-05T11:56:09,473 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:56:09,476 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:56:09,486 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T11:56:09,489 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c, hbase.cluster.distributed=false 2024-12-05T11:56:09,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T11:56:09,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33741 2024-12-05T11:56:09,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33741 2024-12-05T11:56:09,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33741 2024-12-05T11:56:09,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33741 2024-12-05T11:56:09,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33741 2024-12-05T11:56:09,614 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/80666a11a09f:0 server-side Connection retries=45 2024-12-05T11:56:09,616 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T11:56:09,616 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T11:56:09,616 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T11:56:09,616 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T11:56:09,616 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T11:56:09,619 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T11:56:09,621 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T11:56:09,622 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40121 2024-12-05T11:56:09,623 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40121 connecting to ZooKeeper ensemble=127.0.0.1:58051 2024-12-05T11:56:09,624 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:56:09,627 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:56:09,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401210x0, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T11:56:09,638 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:401210x0, quorum=127.0.0.1:58051, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T11:56:09,638 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40121-0x101a6a30e880001 connected 2024-12-05T11:56:09,641 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T11:56:09,648 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T11:56:09,650 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T11:56:09,655 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T11:56:09,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40121 2024-12-05T11:56:09,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40121 2024-12-05T11:56:09,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40121 2024-12-05T11:56:09,657 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40121 2024-12-05T11:56:09,657 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40121 2024-12-05T11:56:09,670 DEBUG [M:0;80666a11a09f:33741 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;80666a11a09f:33741 2024-12-05T11:56:09,671 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/80666a11a09f,33741,1733399768894 2024-12-05T11:56:09,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T11:56:09,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T11:56:09,687 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/80666a11a09f,33741,1733399768894 2024-12-05T11:56:09,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T11:56:09,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:56:09,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:56:09,713 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T11:56:09,714 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/80666a11a09f,33741,1733399768894 from backup master directory 2024-12-05T11:56:09,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/80666a11a09f,33741,1733399768894 2024-12-05T11:56:09,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T11:56:09,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T11:56:09,720 WARN [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T11:56:09,721 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=80666a11a09f,33741,1733399768894 2024-12-05T11:56:09,722 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T11:56:09,723 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T11:56:09,770 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/hbase.id] with ID: ffd85ce0-e522-4ae9-bc47-bc214516dd87 2024-12-05T11:56:09,770 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/.tmp/hbase.id 2024-12-05T11:56:09,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741826_1002 (size=42) 2024-12-05T11:56:10,184 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/.tmp/hbase.id]:[hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/hbase.id] 2024-12-05T11:56:10,233 INFO [master/80666a11a09f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:56:10,238 INFO [master/80666a11a09f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T11:56:10,255 INFO [master/80666a11a09f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-05T11:56:10,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:56:10,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:56:10,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741827_1003 (size=196) 2024-12-05T11:56:10,336 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:56:10,338 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T11:56:10,352 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:10,355 INFO [master/80666a11a09f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T11:56:10,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741828_1004 (size=1189) 2024-12-05T11:56:10,805 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store 2024-12-05T11:56:10,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741829_1005 (size=34) 2024-12-05T11:56:11,231 INFO [master/80666a11a09f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T11:56:11,235 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:56:11,237 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T11:56:11,237 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:56:11,237 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:56:11,239 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T11:56:11,240 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:56:11,240 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:56:11,242 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733399771237Disabling compacts and flushes for region at 1733399771237Disabling writes for close at 1733399771240 (+3 ms)Writing region close event to WAL at 1733399771240Closed at 1733399771240 2024-12-05T11:56:11,246 WARN [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/.initializing 2024-12-05T11:56:11,246 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/WALs/80666a11a09f,33741,1733399768894 2024-12-05T11:56:11,266 INFO [master/80666a11a09f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T11:56:11,285 INFO [master/80666a11a09f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80666a11a09f%2C33741%2C1733399768894, suffix=, logDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/WALs/80666a11a09f,33741,1733399768894, archiveDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/oldWALs, maxLogs=10 2024-12-05T11:56:11,308 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/WALs/80666a11a09f,33741,1733399768894/80666a11a09f%2C33741%2C1733399768894.1733399771291, exclude list is [], retry=0 2024-12-05T11:56:11,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36633,DS-ee3e7562-f485-4c73-a36c-b29c62ebc773,DISK] 2024-12-05T11:56:11,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T11:56:11,359 INFO [master/80666a11a09f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/WALs/80666a11a09f,33741,1733399768894/80666a11a09f%2C33741%2C1733399768894.1733399771291 2024-12-05T11:56:11,359 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46259:46259)] 2024-12-05T11:56:11,360 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:56:11,360 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:56:11,363 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:56:11,364 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:56:11,396 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:56:11,417 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T11:56:11,420 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:11,423 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:56:11,424 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:56:11,427 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T11:56:11,427 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:11,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:56:11,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:56:11,431 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T11:56:11,432 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:11,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:56:11,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:56:11,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T11:56:11,435 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:11,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:56:11,437 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:56:11,440 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:56:11,441 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:56:11,446 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:56:11,446 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:56:11,450 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T11:56:11,454 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:56:11,458 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:56:11,460 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65204331, jitterRate=-0.02837975323200226}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T11:56:11,464 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733399771375Initializing all the Stores at 1733399771376 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733399771377 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399771378 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399771378Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399771378Cleaning up temporary data from old regions at 1733399771446 (+68 ms)Region opened successfully at 1733399771464 (+18 ms) 2024-12-05T11:56:11,466 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T11:56:11,494 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b8984a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80666a11a09f/172.17.0.2:0 2024-12-05T11:56:11,519 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T11:56:11,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T11:56:11,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T11:56:11,535 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T11:56:11,536 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T11:56:11,541 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T11:56:11,541 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T11:56:11,570 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T11:56:11,580 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T11:56:11,627 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T11:56:11,630 INFO [master/80666a11a09f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T11:56:11,633 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T11:56:11,643 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T11:56:11,646 INFO [master/80666a11a09f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T11:56:11,650 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T11:56:11,660 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T11:56:11,662 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T11:56:11,670 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T11:56:11,692 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T11:56:11,701 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T11:56:11,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T11:56:11,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T11:56:11,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:56:11,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:56:11,716 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=80666a11a09f,33741,1733399768894, sessionid=0x101a6a30e880000, setting cluster-up flag (Was=false) 2024-12-05T11:56:11,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:56:11,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:56:11,770 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T11:56:11,774 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80666a11a09f,33741,1733399768894 2024-12-05T11:56:11,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:56:11,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:56:11,820 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T11:56:11,822 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80666a11a09f,33741,1733399768894 2024-12-05T11:56:11,830 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T11:56:11,863 INFO [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(746): ClusterId : ffd85ce0-e522-4ae9-bc47-bc214516dd87 2024-12-05T11:56:11,865 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T11:56:11,879 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T11:56:11,879 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T11:56:11,888 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T11:56:11,889 DEBUG [RS:0;80666a11a09f:40121 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab6ad6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80666a11a09f/172.17.0.2:0 2024-12-05T11:56:11,897 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T11:56:11,904 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;80666a11a09f:40121 2024-12-05T11:56:11,906 INFO [master/80666a11a09f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T11:56:11,907 INFO [RS:0;80666a11a09f:40121 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T11:56:11,907 INFO [RS:0;80666a11a09f:40121 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T11:56:11,907 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T11:56:11,910 INFO [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(2659): reportForDuty to master=80666a11a09f,33741,1733399768894 with port=40121, startcode=1733399769572 2024-12-05T11:56:11,912 INFO [master/80666a11a09f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T11:56:11,916 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 80666a11a09f,33741,1733399768894 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T11:56:11,920 DEBUG [RS:0;80666a11a09f:40121 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T11:56:11,923 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/80666a11a09f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T11:56:11,923 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/80666a11a09f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T11:56:11,923 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/80666a11a09f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T11:56:11,923 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/80666a11a09f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T11:56:11,924 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/80666a11a09f:0, corePoolSize=10, maxPoolSize=10 2024-12-05T11:56:11,924 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:11,924 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/80666a11a09f:0, corePoolSize=2, maxPoolSize=2 2024-12-05T11:56:11,924 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:11,925 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733399801925 2024-12-05T11:56:11,927 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T11:56:11,928 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T11:56:11,929 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T11:56:11,929 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T11:56:11,931 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T11:56:11,932 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T11:56:11,932 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T11:56:11,932 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T11:56:11,933 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:11,936 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T11:56:11,936 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:11,937 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T11:56:11,936 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T11:56:11,937 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T11:56:11,940 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T11:56:11,940 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T11:56:11,945 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.large.0-1733399771941,5,FailOnTimeoutGroup] 2024-12-05T11:56:11,946 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.small.0-1733399771945,5,FailOnTimeoutGroup] 2024-12-05T11:56:11,947 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:11,947 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T11:56:11,948 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:11,949 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:11,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741831_1007 (size=1321) 2024-12-05T11:56:11,955 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35151, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T11:56:11,960 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33741 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 80666a11a09f,40121,1733399769572 2024-12-05T11:56:11,962 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33741 {}] master.ServerManager(517): Registering regionserver=80666a11a09f,40121,1733399769572 2024-12-05T11:56:11,973 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c 2024-12-05T11:56:11,973 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40895 2024-12-05T11:56:11,973 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T11:56:11,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T11:56:11,985 DEBUG [RS:0;80666a11a09f:40121 {}] zookeeper.ZKUtil(111): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80666a11a09f,40121,1733399769572 2024-12-05T11:56:11,986 WARN [RS:0;80666a11a09f:40121 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T11:56:11,986 INFO [RS:0;80666a11a09f:40121 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T11:56:11,986 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/WALs/80666a11a09f,40121,1733399769572 2024-12-05T11:56:11,989 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80666a11a09f,40121,1733399769572] 2024-12-05T11:56:12,011 INFO [RS:0;80666a11a09f:40121 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T11:56:12,023 INFO [RS:0;80666a11a09f:40121 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T11:56:12,028 INFO [RS:0;80666a11a09f:40121 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T11:56:12,028 INFO [RS:0;80666a11a09f:40121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:12,029 INFO [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T11:56:12,034 INFO [RS:0;80666a11a09f:40121 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T11:56:12,036 INFO [RS:0;80666a11a09f:40121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:12,036 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:12,036 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:12,037 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:12,037 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:12,037 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:12,037 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80666a11a09f:0, corePoolSize=2, maxPoolSize=2 2024-12-05T11:56:12,037 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:12,038 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:12,038 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:12,038 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:12,038 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:12,038 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:56:12,038 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80666a11a09f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T11:56:12,039 DEBUG [RS:0;80666a11a09f:40121 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T11:56:12,040 INFO [RS:0;80666a11a09f:40121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:12,040 INFO [RS:0;80666a11a09f:40121 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:12,040 INFO [RS:0;80666a11a09f:40121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:12,040 INFO [RS:0;80666a11a09f:40121 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:12,040 INFO [RS:0;80666a11a09f:40121 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:12,040 INFO [RS:0;80666a11a09f:40121 {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,40121,1733399769572-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T11:56:12,055 INFO [RS:0;80666a11a09f:40121 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T11:56:12,057 INFO [RS:0;80666a11a09f:40121 {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,40121,1733399769572-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:12,057 INFO [RS:0;80666a11a09f:40121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:12,057 INFO [RS:0;80666a11a09f:40121 {}] regionserver.Replication(171): 80666a11a09f,40121,1733399769572 started 2024-12-05T11:56:12,072 INFO [RS:0;80666a11a09f:40121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:12,072 INFO [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1482): Serving as 80666a11a09f,40121,1733399769572, RpcServer on 80666a11a09f/172.17.0.2:40121, sessionid=0x101a6a30e880001 2024-12-05T11:56:12,073 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T11:56:12,073 DEBUG [RS:0;80666a11a09f:40121 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80666a11a09f,40121,1733399769572 2024-12-05T11:56:12,073 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80666a11a09f,40121,1733399769572' 2024-12-05T11:56:12,073 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T11:56:12,074 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T11:56:12,075 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T11:56:12,075 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T11:56:12,075 DEBUG [RS:0;80666a11a09f:40121 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80666a11a09f,40121,1733399769572 2024-12-05T11:56:12,075 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80666a11a09f,40121,1733399769572' 2024-12-05T11:56:12,076 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T11:56:12,076 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T11:56:12,077 DEBUG [RS:0;80666a11a09f:40121 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T11:56:12,077 INFO [RS:0;80666a11a09f:40121 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T11:56:12,077 INFO [RS:0;80666a11a09f:40121 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T11:56:12,185 INFO [RS:0;80666a11a09f:40121 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T11:56:12,190 INFO [RS:0;80666a11a09f:40121 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80666a11a09f%2C40121%2C1733399769572, suffix=, logDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/WALs/80666a11a09f,40121,1733399769572, archiveDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/oldWALs, maxLogs=32 2024-12-05T11:56:12,204 DEBUG [RS:0;80666a11a09f:40121 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/WALs/80666a11a09f,40121,1733399769572/80666a11a09f%2C40121%2C1733399769572.1733399772193, exclude list is [], retry=0 2024-12-05T11:56:12,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36633,DS-ee3e7562-f485-4c73-a36c-b29c62ebc773,DISK] 2024-12-05T11:56:12,212 INFO [RS:0;80666a11a09f:40121 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/WALs/80666a11a09f,40121,1733399769572/80666a11a09f%2C40121%2C1733399769572.1733399772193 2024-12-05T11:56:12,213 DEBUG [RS:0;80666a11a09f:40121 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46259:46259)] 2024-12-05T11:56:12,353 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T11:56:12,354 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c 2024-12-05T11:56:12,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741833_1009 (size=32) 2024-12-05T11:56:12,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:56:12,770 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T11:56:12,772 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T11:56:12,772 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:12,773 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:56:12,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T11:56:12,776 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T11:56:12,777 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:12,778 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:56:12,778 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T11:56:12,780 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T11:56:12,780 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:12,781 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:56:12,782 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T11:56:12,784 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T11:56:12,784 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:12,785 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:56:12,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T11:56:12,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740 2024-12-05T11:56:12,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740 2024-12-05T11:56:12,791 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T11:56:12,791 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T11:56:12,792 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:56:12,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T11:56:12,799 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:56:12,800 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70680945, jitterRate=0.05322815477848053}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:56:12,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733399772767Initializing all the Stores at 1733399772769 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733399772769Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733399772769Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399772769Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733399772770 (+1 ms)Cleaning up temporary data from old regions at 1733399772791 (+21 ms)Region opened successfully at 1733399772803 (+12 ms) 2024-12-05T11:56:12,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T11:56:12,803 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T11:56:12,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T11:56:12,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T11:56:12,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T11:56:12,805 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T11:56:12,805 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733399772803Disabling compacts and flushes for region at 1733399772803Disabling writes for close at 1733399772803Writing region close event to WAL at 1733399772804 (+1 ms)Closed at 1733399772805 (+1 ms) 2024-12-05T11:56:12,809 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T11:56:12,809 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T11:56:12,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T11:56:12,822 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T11:56:12,825 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T11:56:12,980 DEBUG [80666a11a09f:33741 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T11:56:12,992 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:56:12,998 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80666a11a09f,40121,1733399769572, state=OPENING 2024-12-05T11:56:13,052 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T11:56:13,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:56:13,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:56:13,064 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T11:56:13,064 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T11:56:13,067 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T11:56:13,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:56:13,253 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T11:56:13,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50653, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T11:56:13,266 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T11:56:13,266 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T11:56:13,267 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T11:56:13,271 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80666a11a09f%2C40121%2C1733399769572.meta, suffix=.meta, logDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/WALs/80666a11a09f,40121,1733399769572, archiveDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/oldWALs, maxLogs=32 2024-12-05T11:56:13,284 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/WALs/80666a11a09f,40121,1733399769572/80666a11a09f%2C40121%2C1733399769572.meta.1733399773272.meta, exclude list is [], retry=0 2024-12-05T11:56:13,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36633,DS-ee3e7562-f485-4c73-a36c-b29c62ebc773,DISK] 2024-12-05T11:56:13,290 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/WALs/80666a11a09f,40121,1733399769572/80666a11a09f%2C40121%2C1733399769572.meta.1733399773272.meta 2024-12-05T11:56:13,291 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46259:46259)] 2024-12-05T11:56:13,291 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:56:13,293 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T11:56:13,295 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T11:56:13,299 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T11:56:13,303 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T11:56:13,304 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:56:13,304 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T11:56:13,305 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T11:56:13,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T11:56:13,309 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T11:56:13,309 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:13,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:56:13,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T11:56:13,312 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T11:56:13,312 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:13,313 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:56:13,313 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T11:56:13,315 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T11:56:13,315 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:13,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:56:13,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T11:56:13,317 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T11:56:13,317 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:13,318 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:56:13,319 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T11:56:13,320 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740 2024-12-05T11:56:13,322 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740 2024-12-05T11:56:13,325 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T11:56:13,325 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T11:56:13,326 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:56:13,328 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T11:56:13,330 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66747001, jitterRate=-0.005392178893089294}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:56:13,330 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T11:56:13,331 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733399773305Writing region info on filesystem at 1733399773305Initializing all the Stores at 1733399773307 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733399773307Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733399773307Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399773307Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733399773307Cleaning up temporary data from old regions at 1733399773325 (+18 ms)Running coprocessor post-open hooks at 1733399773330 (+5 ms)Region opened successfully at 1733399773331 (+1 ms) 2024-12-05T11:56:13,337 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733399773246 2024-12-05T11:56:13,347 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T11:56:13,348 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T11:56:13,350 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:56:13,352 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80666a11a09f,40121,1733399769572, state=OPEN 2024-12-05T11:56:13,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T11:56:13,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T11:56:13,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T11:56:13,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T11:56:13,379 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:13,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T11:56:13,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=80666a11a09f,40121,1733399769572 in 309 msec 2024-12-05T11:56:13,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T11:56:13,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 574 msec 2024-12-05T11:56:13,395 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T11:56:13,395 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T11:56:13,415 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:13,417 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:13,435 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:13,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51753, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:13,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6000 sec 2024-12-05T11:56:13,460 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733399773460, completionTime=-1 2024-12-05T11:56:13,462 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T11:56:13,463 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T11:56:13,487 INFO [master/80666a11a09f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-05T11:56:13,487 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733399833487 2024-12-05T11:56:13,487 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733399893487 2024-12-05T11:56:13,487 INFO [master/80666a11a09f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-05T11:56:13,489 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,33741,1733399768894-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:13,490 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,33741,1733399768894-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:13,490 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,33741,1733399768894-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:13,491 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-80666a11a09f:33741, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:13,492 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:13,492 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:13,499 DEBUG [master/80666a11a09f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T11:56:13,517 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.796sec 2024-12-05T11:56:13,518 INFO [master/80666a11a09f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T11:56:13,519 INFO [master/80666a11a09f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T11:56:13,520 INFO [master/80666a11a09f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T11:56:13,520 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T11:56:13,521 INFO [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T11:56:13,521 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,33741,1733399768894-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T11:56:13,522 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,33741,1733399768894-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T11:56:13,530 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T11:56:13,531 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T11:56:13,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,33741,1733399768894-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T11:56:13,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:13,579 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T11:56:13,579 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T11:56:13,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:13,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:13,591 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:13,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:13,601 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:13,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:13,604 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:13,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:13,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=80666a11a09f,33741,1733399768894 2024-12-05T11:56:13,645 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=223, ProcessCount=11, AvailableMemoryMB=7423 2024-12-05T11:56:13,653 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:56:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 80666a11a09f,33741,1733399768894 2024-12-05T11:56:13,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f523d87 2024-12-05T11:56:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:56:13,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:56:13,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:56:13,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:56:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T11:56:13,702 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T11:56:13,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-05T11:56:13,708 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:13,711 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T11:56:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T11:56:13,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741835_1011 (size=959) 2024-12-05T11:56:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T11:56:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T11:56:14,130 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c 2024-12-05T11:56:14,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741836_1012 (size=53) 2024-12-05T11:56:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T11:56:14,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:56:14,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 41e385893d26f932b2fcba15c0b7f150, disabling compactions & flushes 2024-12-05T11:56:14,543 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:14,543 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:14,543 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. after waiting 0 ms 2024-12-05T11:56:14,543 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:14,543 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:14,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 41e385893d26f932b2fcba15c0b7f150: Waiting for close lock at 1733399774542Disabling compacts and flushes for region at 1733399774542Disabling writes for close at 1733399774543 (+1 ms)Writing region close event to WAL at 1733399774543Closed at 1733399774543 2024-12-05T11:56:14,549 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T11:56:14,555 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733399774549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733399774549"}]},"ts":"1733399774549"} 2024-12-05T11:56:14,560 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T11:56:14,562 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T11:56:14,564 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399774562"}]},"ts":"1733399774562"} 2024-12-05T11:56:14,568 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T11:56:14,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41e385893d26f932b2fcba15c0b7f150, ASSIGN}] 2024-12-05T11:56:14,572 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41e385893d26f932b2fcba15c0b7f150, ASSIGN 2024-12-05T11:56:14,574 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=41e385893d26f932b2fcba15c0b7f150, ASSIGN; state=OFFLINE, location=80666a11a09f,40121,1733399769572; forceNewPlan=false, retain=false 2024-12-05T11:56:14,727 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=41e385893d26f932b2fcba15c0b7f150, regionState=OPENING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:56:14,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=41e385893d26f932b2fcba15c0b7f150, ASSIGN because future has completed 2024-12-05T11:56:14,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:56:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T11:56:14,898 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:14,898 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:56:14,899 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,899 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:56:14,899 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,899 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,902 INFO [StoreOpener-41e385893d26f932b2fcba15c0b7f150-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,905 INFO [StoreOpener-41e385893d26f932b2fcba15c0b7f150-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41e385893d26f932b2fcba15c0b7f150 columnFamilyName A 2024-12-05T11:56:14,905 DEBUG [StoreOpener-41e385893d26f932b2fcba15c0b7f150-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:14,906 INFO [StoreOpener-41e385893d26f932b2fcba15c0b7f150-1 {}] regionserver.HStore(327): Store=41e385893d26f932b2fcba15c0b7f150/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:56:14,906 INFO [StoreOpener-41e385893d26f932b2fcba15c0b7f150-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,909 INFO [StoreOpener-41e385893d26f932b2fcba15c0b7f150-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41e385893d26f932b2fcba15c0b7f150 columnFamilyName B 2024-12-05T11:56:14,909 DEBUG [StoreOpener-41e385893d26f932b2fcba15c0b7f150-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:14,910 INFO [StoreOpener-41e385893d26f932b2fcba15c0b7f150-1 {}] regionserver.HStore(327): Store=41e385893d26f932b2fcba15c0b7f150/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:56:14,910 INFO [StoreOpener-41e385893d26f932b2fcba15c0b7f150-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,912 INFO [StoreOpener-41e385893d26f932b2fcba15c0b7f150-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41e385893d26f932b2fcba15c0b7f150 columnFamilyName C 2024-12-05T11:56:14,913 DEBUG [StoreOpener-41e385893d26f932b2fcba15c0b7f150-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:14,914 INFO [StoreOpener-41e385893d26f932b2fcba15c0b7f150-1 {}] regionserver.HStore(327): Store=41e385893d26f932b2fcba15c0b7f150/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:56:14,914 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,916 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,917 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,919 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,919 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,920 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:56:14,922 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,925 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:56:14,926 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 41e385893d26f932b2fcba15c0b7f150; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63725749, jitterRate=-0.050412341952323914}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:56:14,926 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:14,927 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 41e385893d26f932b2fcba15c0b7f150: Running coprocessor pre-open hook at 1733399774900Writing region info on filesystem at 1733399774900Initializing all the Stores at 1733399774902 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399774902Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399774902Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399774902Cleaning up temporary data from old regions at 1733399774919 (+17 ms)Running coprocessor post-open hooks at 1733399774926 (+7 ms)Region opened successfully at 1733399774927 (+1 ms) 2024-12-05T11:56:14,929 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., pid=6, masterSystemTime=1733399774889 2024-12-05T11:56:14,932 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:14,932 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:14,933 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=41e385893d26f932b2fcba15c0b7f150, regionState=OPEN, openSeqNum=2, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:56:14,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:56:14,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T11:56:14,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 in 204 msec 2024-12-05T11:56:14,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T11:56:14,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41e385893d26f932b2fcba15c0b7f150, ASSIGN in 374 msec 2024-12-05T11:56:14,950 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T11:56:14,950 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399774950"}]},"ts":"1733399774950"} 2024-12-05T11:56:14,953 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T11:56:14,955 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T11:56:14,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2610 sec 2024-12-05T11:56:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T11:56:15,861 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:15,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:15,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:15,875 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253618fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:15,878 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67dec8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,880 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:15,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:15,887 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:56:15,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 80666a11a09f,33741,1733399768894 2024-12-05T11:56:15,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1df1000 2024-12-05T11:56:15,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:56:15,893 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:56:15,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:15,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:15,898 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d21add1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:15,901 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:15,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:15,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:15,905 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675bb8fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:15,908 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:15,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b21050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:15,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:15,913 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:15,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:15,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:15,917 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:15,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:15,920 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:15,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:15,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:15,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:15,923 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df7e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:15,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:15,928 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:15,930 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5904dd5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:15,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:15,933 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b036a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:15,936 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:15,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6675595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:15,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:15,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4157de12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:15,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e021fe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:15,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:15,945 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:15,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:15,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@905b4d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:15,947 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526369d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:15,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:15,950 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ce558b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:15,952 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1ec368, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:15,959 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:15,959 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:15,959 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:15,959 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:15,960 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:15,960 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:15,960 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:15,960 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:15,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:15,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:15,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:15,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:15,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:15,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:15,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:15,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:56:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:56:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:56:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:56:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:56:15,975 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:56:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:56:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:56:15,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:15,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:15,979 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:56:15,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-05T11:56:16,002 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:16,006 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:16,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:56:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:16,061 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:16,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T11:56:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:56:16,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:16,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/9c52c54b5eb1464196ddbcf5200ab053 is 50, key is test_row_0/A:col10/1733399776029/Put/seqid=0 2024-12-05T11:56:16,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:16,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:16,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:16,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741837_1013 (size=12001) 2024-12-05T11:56:16,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/9c52c54b5eb1464196ddbcf5200ab053 2024-12-05T11:56:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399836215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399836215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399836246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399836247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399836247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:56:16,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/06fe62d13afa4e9ea49003a7c404913b is 50, key is test_row_0/B:col10/1733399776029/Put/seqid=0 2024-12-05T11:56:16,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:16,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:16,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:16,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:16,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741838_1014 (size=12001) 2024-12-05T11:56:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/06fe62d13afa4e9ea49003a7c404913b 2024-12-05T11:56:16,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399836388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399836389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399836389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399836389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399836390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/996a80e8380f40f8ac97588d711d948e is 50, key is test_row_0/C:col10/1733399776029/Put/seqid=0 2024-12-05T11:56:16,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741839_1015 (size=12001) 2024-12-05T11:56:16,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/996a80e8380f40f8ac97588d711d948e 2024-12-05T11:56:16,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/9c52c54b5eb1464196ddbcf5200ab053 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/9c52c54b5eb1464196ddbcf5200ab053 2024-12-05T11:56:16,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/9c52c54b5eb1464196ddbcf5200ab053, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T11:56:16,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:16,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/06fe62d13afa4e9ea49003a7c404913b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/06fe62d13afa4e9ea49003a7c404913b 2024-12-05T11:56:16,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:16,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:16,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/06fe62d13afa4e9ea49003a7c404913b, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T11:56:16,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/996a80e8380f40f8ac97588d711d948e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/996a80e8380f40f8ac97588d711d948e 2024-12-05T11:56:16,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/996a80e8380f40f8ac97588d711d948e, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T11:56:16,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 41e385893d26f932b2fcba15c0b7f150 in 511ms, sequenceid=14, compaction requested=false 2024-12-05T11:56:16,581 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T11:56:16,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:16,606 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:16,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-05T11:56:16,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/638a7f1f8ab640118ad20abbdce366e5 is 50, key is test_row_0/A:col10/1733399776600/Put/seqid=0 2024-12-05T11:56:16,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399836632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399836632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:56:16,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741840_1016 (size=16681) 2024-12-05T11:56:16,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/638a7f1f8ab640118ad20abbdce366e5 2024-12-05T11:56:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399836637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399836636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399836640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:16,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:16,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/4c7de984108443c2a259681bb29ed837 is 50, key is test_row_0/B:col10/1733399776600/Put/seqid=0 2024-12-05T11:56:16,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741841_1017 (size=12001) 2024-12-05T11:56:16,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/4c7de984108443c2a259681bb29ed837 2024-12-05T11:56:16,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399836748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399836749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/38aabd8152b24cf3b047d4c9782fcad5 is 50, key is test_row_0/C:col10/1733399776600/Put/seqid=0 2024-12-05T11:56:16,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399836769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399836770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399836770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741842_1018 (size=12001) 2024-12-05T11:56:16,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:16,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:16,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:16,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:16,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:16,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399836959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399836968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399836980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399836980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:16,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399836980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:16,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:16,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:17,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:17,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:56:17,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:17,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:17,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/38aabd8152b24cf3b047d4c9782fcad5 2024-12-05T11:56:17,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/638a7f1f8ab640118ad20abbdce366e5 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/638a7f1f8ab640118ad20abbdce366e5 2024-12-05T11:56:17,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/638a7f1f8ab640118ad20abbdce366e5, entries=250, sequenceid=41, filesize=16.3 K 2024-12-05T11:56:17,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/4c7de984108443c2a259681bb29ed837 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/4c7de984108443c2a259681bb29ed837 2024-12-05T11:56:17,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/4c7de984108443c2a259681bb29ed837, entries=150, sequenceid=41, filesize=11.7 K 2024-12-05T11:56:17,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/38aabd8152b24cf3b047d4c9782fcad5 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/38aabd8152b24cf3b047d4c9782fcad5 2024-12-05T11:56:17,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/38aabd8152b24cf3b047d4c9782fcad5, entries=150, sequenceid=41, filesize=11.7 K 2024-12-05T11:56:17,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 41e385893d26f932b2fcba15c0b7f150 in 662ms, sequenceid=41, compaction requested=false 2024-12-05T11:56:17,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:17,287 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:17,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T11:56:17,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/42bef86dda1e4347ad59a7907143d74a is 50, key is test_row_0/A:col10/1733399777284/Put/seqid=0 2024-12-05T11:56:17,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741843_1019 (size=14341) 2024-12-05T11:56:17,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/42bef86dda1e4347ad59a7907143d74a 2024-12-05T11:56:17,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:17,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:17,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/374d7ce1910d492a8193e42e31bf9bb6 is 50, key is test_row_0/B:col10/1733399777284/Put/seqid=0 2024-12-05T11:56:17,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741844_1020 (size=12001) 2024-12-05T11:56:17,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/374d7ce1910d492a8193e42e31bf9bb6 2024-12-05T11:56:17,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/c53d8ee1b7884d0ca6fb0b205bc621db is 50, key is test_row_0/C:col10/1733399777284/Put/seqid=0 2024-12-05T11:56:17,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741845_1021 (size=12001) 2024-12-05T11:56:17,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/c53d8ee1b7884d0ca6fb0b205bc621db 2024-12-05T11:56:17,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399837414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399837419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399837423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/42bef86dda1e4347ad59a7907143d74a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/42bef86dda1e4347ad59a7907143d74a 2024-12-05T11:56:17,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399837430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399837432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/42bef86dda1e4347ad59a7907143d74a, entries=200, sequenceid=52, filesize=14.0 K 2024-12-05T11:56:17,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/374d7ce1910d492a8193e42e31bf9bb6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/374d7ce1910d492a8193e42e31bf9bb6 2024-12-05T11:56:17,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/374d7ce1910d492a8193e42e31bf9bb6, entries=150, sequenceid=52, filesize=11.7 K 2024-12-05T11:56:17,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/c53d8ee1b7884d0ca6fb0b205bc621db as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/c53d8ee1b7884d0ca6fb0b205bc621db 2024-12-05T11:56:17,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/c53d8ee1b7884d0ca6fb0b205bc621db, entries=150, sequenceid=52, filesize=11.7 K 2024-12-05T11:56:17,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 41e385893d26f932b2fcba15c0b7f150 in 212ms, sequenceid=52, compaction requested=true 2024-12-05T11:56:17,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:17,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:17,504 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:17,504 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:17,508 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:17,510 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/B is initiating minor compaction (all files) 2024-12-05T11:56:17,511 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/B in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,511 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/06fe62d13afa4e9ea49003a7c404913b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/4c7de984108443c2a259681bb29ed837, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/374d7ce1910d492a8193e42e31bf9bb6] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=35.2 K 2024-12-05T11:56:17,513 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 06fe62d13afa4e9ea49003a7c404913b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733399775983 2024-12-05T11:56:17,515 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:17,515 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c7de984108443c2a259681bb29ed837, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733399776229 2024-12-05T11:56:17,515 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/A is initiating minor compaction (all files) 2024-12-05T11:56:17,515 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/A in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,516 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/9c52c54b5eb1464196ddbcf5200ab053, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/638a7f1f8ab640118ad20abbdce366e5, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/42bef86dda1e4347ad59a7907143d74a] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=42.0 K 2024-12-05T11:56:17,516 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 374d7ce1910d492a8193e42e31bf9bb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399776627 2024-12-05T11:56:17,517 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c52c54b5eb1464196ddbcf5200ab053, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733399775983 2024-12-05T11:56:17,520 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 638a7f1f8ab640118ad20abbdce366e5, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733399776163 2024-12-05T11:56:17,521 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42bef86dda1e4347ad59a7907143d74a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399776627 2024-12-05T11:56:17,543 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:17,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-05T11:56:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:17,552 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#B#compaction#9 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:17,554 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/6853adb7d3a54aa2b1096e8ceac0e5f3 is 50, key is test_row_0/B:col10/1733399777284/Put/seqid=0 2024-12-05T11:56:17,554 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#A#compaction#10 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:17,555 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/2b2c56fc9d3a4c70bf235dca13671c5b is 50, key is test_row_0/A:col10/1733399777284/Put/seqid=0 2024-12-05T11:56:17,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/1b1d932fb18c4dd8860bb3b91386c466 is 50, key is test_row_0/A:col10/1733399777422/Put/seqid=0 2024-12-05T11:56:17,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399837565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399837565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399837568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399837570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399837571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741846_1022 (size=12104) 2024-12-05T11:56:17,605 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/2b2c56fc9d3a4c70bf235dca13671c5b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/2b2c56fc9d3a4c70bf235dca13671c5b 2024-12-05T11:56:17,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741848_1024 (size=12104) 2024-12-05T11:56:17,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741847_1023 (size=14341) 2024-12-05T11:56:17,632 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/6853adb7d3a54aa2b1096e8ceac0e5f3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/6853adb7d3a54aa2b1096e8ceac0e5f3 2024-12-05T11:56:17,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:17,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,656 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/B of 41e385893d26f932b2fcba15c0b7f150 into 6853adb7d3a54aa2b1096e8ceac0e5f3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:17,657 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:17,657 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/A of 41e385893d26f932b2fcba15c0b7f150 into 2b2c56fc9d3a4c70bf235dca13671c5b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:17,657 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:17,659 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/A, priority=13, startTime=1733399777501; duration=0sec 2024-12-05T11:56:17,659 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/B, priority=13, startTime=1733399777504; duration=0sec 2024-12-05T11:56:17,660 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:17,660 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:A 2024-12-05T11:56:17,661 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:17,664 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:17,664 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:B 2024-12-05T11:56:17,666 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:17,666 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/C is initiating minor compaction (all files) 2024-12-05T11:56:17,666 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/C in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,667 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/996a80e8380f40f8ac97588d711d948e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/38aabd8152b24cf3b047d4c9782fcad5, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/c53d8ee1b7884d0ca6fb0b205bc621db] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=35.2 K 2024-12-05T11:56:17,668 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 996a80e8380f40f8ac97588d711d948e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733399775983 2024-12-05T11:56:17,669 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38aabd8152b24cf3b047d4c9782fcad5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733399776229 2024-12-05T11:56:17,672 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c53d8ee1b7884d0ca6fb0b205bc621db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399776627 2024-12-05T11:56:17,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399837679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399837679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399837688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399837689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399837689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,716 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:17,717 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/2550e03d4e5f4298880a8f4184c2c84b is 50, key is test_row_0/C:col10/1733399777284/Put/seqid=0 2024-12-05T11:56:17,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741849_1025 (size=12104) 2024-12-05T11:56:17,773 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/2550e03d4e5f4298880a8f4184c2c84b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/2550e03d4e5f4298880a8f4184c2c84b 2024-12-05T11:56:17,788 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/C of 41e385893d26f932b2fcba15c0b7f150 into 2550e03d4e5f4298880a8f4184c2c84b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:17,788 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:17,788 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/C, priority=13, startTime=1733399777504; duration=0sec 2024-12-05T11:56:17,788 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:17,788 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:C 2024-12-05T11:56:17,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399837889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399837898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399837900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399837909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:17,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399837909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:17,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:17,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:17,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:17,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:18,006 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T11:56:18,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/1b1d932fb18c4dd8860bb3b91386c466 2024-12-05T11:56:18,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/94c9d54b55be4537b07e26ab4e98de0e is 50, key is test_row_0/B:col10/1733399777422/Put/seqid=0 2024-12-05T11:56:18,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741850_1026 (size=12001) 2024-12-05T11:56:18,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/94c9d54b55be4537b07e26ab4e98de0e 2024-12-05T11:56:18,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/e6769e4d65984cf0b94c0cbeb3fd426e is 50, key is test_row_0/C:col10/1733399777422/Put/seqid=0 2024-12-05T11:56:18,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:18,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:18,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:18,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741851_1027 (size=12001) 2024-12-05T11:56:18,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/e6769e4d65984cf0b94c0cbeb3fd426e 2024-12-05T11:56:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/1b1d932fb18c4dd8860bb3b91386c466 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/1b1d932fb18c4dd8860bb3b91386c466 2024-12-05T11:56:18,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/1b1d932fb18c4dd8860bb3b91386c466, entries=200, sequenceid=79, filesize=14.0 K 2024-12-05T11:56:18,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/94c9d54b55be4537b07e26ab4e98de0e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/94c9d54b55be4537b07e26ab4e98de0e 2024-12-05T11:56:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:56:18,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/94c9d54b55be4537b07e26ab4e98de0e, entries=150, sequenceid=79, filesize=11.7 K 2024-12-05T11:56:18,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/e6769e4d65984cf0b94c0cbeb3fd426e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e6769e4d65984cf0b94c0cbeb3fd426e 2024-12-05T11:56:18,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399838198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399838208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e6769e4d65984cf0b94c0cbeb3fd426e, entries=150, sequenceid=79, filesize=11.7 K 2024-12-05T11:56:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 41e385893d26f932b2fcba15c0b7f150 in 668ms, sequenceid=79, compaction requested=false 2024-12-05T11:56:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:18,221 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:18,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T11:56:18,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/540fdfbcbecd432698cc96cb00866aa6 is 50, key is test_row_0/A:col10/1733399778221/Put/seqid=0 2024-12-05T11:56:18,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741852_1028 (size=14341) 2024-12-05T11:56:18,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:18,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:18,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:18,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:18,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:18,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:18,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/540fdfbcbecd432698cc96cb00866aa6 2024-12-05T11:56:18,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399838293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399838299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399838302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/fb57f307a5c44cf09007e9963fcb613c is 50, key is test_row_0/B:col10/1733399778221/Put/seqid=0 2024-12-05T11:56:18,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741853_1029 (size=12001) 2024-12-05T11:56:18,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/fb57f307a5c44cf09007e9963fcb613c 2024-12-05T11:56:18,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/4f24d51eb4e8419db5d420131de99a61 is 50, key is test_row_0/C:col10/1733399778221/Put/seqid=0 2024-12-05T11:56:18,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399838408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399838418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399838418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741854_1030 (size=12001) 2024-12-05T11:56:18,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:18,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/4f24d51eb4e8419db5d420131de99a61 2024-12-05T11:56:18,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:18,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:18,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:18,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:18,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:18,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/540fdfbcbecd432698cc96cb00866aa6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/540fdfbcbecd432698cc96cb00866aa6 2024-12-05T11:56:18,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/540fdfbcbecd432698cc96cb00866aa6, entries=200, sequenceid=94, filesize=14.0 K 2024-12-05T11:56:18,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/fb57f307a5c44cf09007e9963fcb613c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/fb57f307a5c44cf09007e9963fcb613c 2024-12-05T11:56:18,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/fb57f307a5c44cf09007e9963fcb613c, entries=150, sequenceid=94, filesize=11.7 K 2024-12-05T11:56:18,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/4f24d51eb4e8419db5d420131de99a61 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/4f24d51eb4e8419db5d420131de99a61 2024-12-05T11:56:18,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/4f24d51eb4e8419db5d420131de99a61, entries=150, sequenceid=94, filesize=11.7 K 2024-12-05T11:56:18,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 41e385893d26f932b2fcba15c0b7f150 in 280ms, sequenceid=94, compaction requested=true 2024-12-05T11:56:18,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:18,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:18,502 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:18,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:18,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:18,503 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:18,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:18,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:18,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:18,504 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:18,505 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/A is initiating minor compaction (all files) 2024-12-05T11:56:18,505 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/A in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:18,505 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/2b2c56fc9d3a4c70bf235dca13671c5b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/1b1d932fb18c4dd8860bb3b91386c466, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/540fdfbcbecd432698cc96cb00866aa6] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=39.8 K 2024-12-05T11:56:18,505 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:18,506 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/B is initiating minor compaction (all files) 2024-12-05T11:56:18,506 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/B in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:18,506 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/6853adb7d3a54aa2b1096e8ceac0e5f3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/94c9d54b55be4537b07e26ab4e98de0e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/fb57f307a5c44cf09007e9963fcb613c] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=35.3 K 2024-12-05T11:56:18,508 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b2c56fc9d3a4c70bf235dca13671c5b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399776627 2024-12-05T11:56:18,508 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6853adb7d3a54aa2b1096e8ceac0e5f3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399776627 2024-12-05T11:56:18,509 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b1d932fb18c4dd8860bb3b91386c466, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733399777417 2024-12-05T11:56:18,510 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 94c9d54b55be4537b07e26ab4e98de0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733399777422 2024-12-05T11:56:18,510 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 540fdfbcbecd432698cc96cb00866aa6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399777564 2024-12-05T11:56:18,511 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting fb57f307a5c44cf09007e9963fcb613c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399777564 2024-12-05T11:56:18,536 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#A#compaction#18 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:18,537 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:18,537 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/8118d40a82694c178f222bcc12d5d8c2 is 50, key is test_row_0/A:col10/1733399778221/Put/seqid=0 2024-12-05T11:56:18,537 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/f08f466039f441238b0b24e56b5b513b is 50, key is test_row_0/B:col10/1733399778221/Put/seqid=0 2024-12-05T11:56:18,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741855_1031 (size=12207) 2024-12-05T11:56:18,582 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/f08f466039f441238b0b24e56b5b513b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f08f466039f441238b0b24e56b5b513b 2024-12-05T11:56:18,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:56:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:18,585 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T11:56:18,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741856_1032 (size=12207) 2024-12-05T11:56:18,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/5924132f784f416fa59279e9114cfd70 is 50, key is test_row_0/A:col10/1733399778284/Put/seqid=0 2024-12-05T11:56:18,607 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/8118d40a82694c178f222bcc12d5d8c2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8118d40a82694c178f222bcc12d5d8c2 2024-12-05T11:56:18,627 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/A of 41e385893d26f932b2fcba15c0b7f150 into 8118d40a82694c178f222bcc12d5d8c2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:18,628 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/B of 41e385893d26f932b2fcba15c0b7f150 into f08f466039f441238b0b24e56b5b513b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:18,628 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:18,628 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/B, priority=13, startTime=1733399778502; duration=0sec 2024-12-05T11:56:18,628 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:18,628 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/A, priority=13, startTime=1733399778502; duration=0sec 2024-12-05T11:56:18,628 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:18,628 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:B 2024-12-05T11:56:18,628 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:18,628 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:18,628 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:A 2024-12-05T11:56:18,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741857_1033 (size=12001) 2024-12-05T11:56:18,631 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:18,631 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/C is initiating minor compaction (all files) 2024-12-05T11:56:18,631 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/C in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:18,631 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/2550e03d4e5f4298880a8f4184c2c84b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e6769e4d65984cf0b94c0cbeb3fd426e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/4f24d51eb4e8419db5d420131de99a61] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=35.3 K 2024-12-05T11:56:18,632 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/5924132f784f416fa59279e9114cfd70 2024-12-05T11:56:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:18,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:18,632 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2550e03d4e5f4298880a8f4184c2c84b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399776627 2024-12-05T11:56:18,634 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting e6769e4d65984cf0b94c0cbeb3fd426e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733399777422 2024-12-05T11:56:18,638 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f24d51eb4e8419db5d420131de99a61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399777564 2024-12-05T11:56:18,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/63791b4e5d274c17bc8ef04b50490ea6 is 50, key is test_row_0/B:col10/1733399778284/Put/seqid=0 2024-12-05T11:56:18,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399838658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399838661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399838664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,673 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#C#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:18,676 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/af5d094374d64f85a6669bce8c8ce29f is 50, key is test_row_0/C:col10/1733399778221/Put/seqid=0 2024-12-05T11:56:18,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741858_1034 (size=12001) 2024-12-05T11:56:18,699 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/63791b4e5d274c17bc8ef04b50490ea6 2024-12-05T11:56:18,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741859_1035 (size=12207) 2024-12-05T11:56:18,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399838709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,719 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/af5d094374d64f85a6669bce8c8ce29f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/af5d094374d64f85a6669bce8c8ce29f 2024-12-05T11:56:18,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399838719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/6991b238a930405bad403f2541b08749 is 50, key is test_row_0/C:col10/1733399778284/Put/seqid=0 2024-12-05T11:56:18,739 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/C of 41e385893d26f932b2fcba15c0b7f150 into af5d094374d64f85a6669bce8c8ce29f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:18,739 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:18,739 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/C, priority=13, startTime=1733399778503; duration=0sec 2024-12-05T11:56:18,740 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:18,740 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:C 2024-12-05T11:56:18,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741860_1036 (size=12001) 2024-12-05T11:56:18,761 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/6991b238a930405bad403f2541b08749 2024-12-05T11:56:18,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399838768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/5924132f784f416fa59279e9114cfd70 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5924132f784f416fa59279e9114cfd70 2024-12-05T11:56:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399838778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399838778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:18,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:18,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:18,792 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5924132f784f416fa59279e9114cfd70, entries=150, sequenceid=119, filesize=11.7 K 2024-12-05T11:56:18,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/63791b4e5d274c17bc8ef04b50490ea6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/63791b4e5d274c17bc8ef04b50490ea6 2024-12-05T11:56:18,807 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/63791b4e5d274c17bc8ef04b50490ea6, entries=150, sequenceid=119, filesize=11.7 K 2024-12-05T11:56:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/6991b238a930405bad403f2541b08749 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/6991b238a930405bad403f2541b08749 2024-12-05T11:56:18,815 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:56:18,832 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/6991b238a930405bad403f2541b08749, entries=150, sequenceid=119, filesize=11.7 K 2024-12-05T11:56:18,835 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 41e385893d26f932b2fcba15c0b7f150 in 249ms, sequenceid=119, compaction requested=false 2024-12-05T11:56:18,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:18,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:18,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-05T11:56:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-05T11:56:18,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T11:56:18,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8290 sec 2024-12-05T11:56:18,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.8490 sec 2024-12-05T11:56:18,981 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:18,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T11:56:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:19,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/4d60b24621cc458393523d972fa1e6af is 50, key is test_row_0/A:col10/1733399778979/Put/seqid=0 2024-12-05T11:56:19,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741861_1037 (size=14491) 2024-12-05T11:56:19,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/4d60b24621cc458393523d972fa1e6af 2024-12-05T11:56:19,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/70c8def2bd2c4c1ebe889a19776b76bb is 50, key is test_row_0/B:col10/1733399778979/Put/seqid=0 2024-12-05T11:56:19,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399839054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399839054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399839059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741862_1038 (size=12101) 2024-12-05T11:56:19,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/70c8def2bd2c4c1ebe889a19776b76bb 2024-12-05T11:56:19,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/e08a989c584d4333b06a69ff39a3bc89 is 50, key is test_row_0/C:col10/1733399778979/Put/seqid=0 2024-12-05T11:56:19,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741863_1039 (size=12101) 2024-12-05T11:56:19,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/e08a989c584d4333b06a69ff39a3bc89 2024-12-05T11:56:19,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399839168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399839169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399839169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/4d60b24621cc458393523d972fa1e6af as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/4d60b24621cc458393523d972fa1e6af 2024-12-05T11:56:19,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/4d60b24621cc458393523d972fa1e6af, entries=200, sequenceid=134, filesize=14.2 K 2024-12-05T11:56:19,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/70c8def2bd2c4c1ebe889a19776b76bb as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/70c8def2bd2c4c1ebe889a19776b76bb 2024-12-05T11:56:19,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/70c8def2bd2c4c1ebe889a19776b76bb, entries=150, sequenceid=134, filesize=11.8 K 2024-12-05T11:56:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/e08a989c584d4333b06a69ff39a3bc89 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e08a989c584d4333b06a69ff39a3bc89 2024-12-05T11:56:19,212 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T11:56:19,212 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T11:56:19,214 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T11:56:19,214 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T11:56:19,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e08a989c584d4333b06a69ff39a3bc89, entries=150, sequenceid=134, filesize=11.8 K 2024-12-05T11:56:19,215 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T11:56:19,215 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T11:56:19,216 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T11:56:19,216 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T11:56:19,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 41e385893d26f932b2fcba15c0b7f150 in 235ms, sequenceid=134, compaction requested=true 2024-12-05T11:56:19,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:19,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:19,217 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:19,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:19,217 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:19,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:19,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:19,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:19,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:19,219 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:19,219 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/A is initiating minor compaction (all files) 2024-12-05T11:56:19,219 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/A in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:19,220 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8118d40a82694c178f222bcc12d5d8c2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5924132f784f416fa59279e9114cfd70, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/4d60b24621cc458393523d972fa1e6af] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=37.8 K 2024-12-05T11:56:19,220 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:19,220 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/B is initiating minor compaction (all files) 2024-12-05T11:56:19,220 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/B in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:19,220 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f08f466039f441238b0b24e56b5b513b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/63791b4e5d274c17bc8ef04b50490ea6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/70c8def2bd2c4c1ebe889a19776b76bb] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=35.5 K 2024-12-05T11:56:19,221 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8118d40a82694c178f222bcc12d5d8c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399777564 2024-12-05T11:56:19,222 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting f08f466039f441238b0b24e56b5b513b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399777564 2024-12-05T11:56:19,223 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5924132f784f416fa59279e9114cfd70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733399778284 2024-12-05T11:56:19,224 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 63791b4e5d274c17bc8ef04b50490ea6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733399778284 2024-12-05T11:56:19,224 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d60b24621cc458393523d972fa1e6af, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399778644 2024-12-05T11:56:19,225 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 70c8def2bd2c4c1ebe889a19776b76bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399778644 2024-12-05T11:56:19,242 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:19,243 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/be69616aa13a487daecd3697973abc1d is 50, key is test_row_0/B:col10/1733399778979/Put/seqid=0 2024-12-05T11:56:19,246 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:19,247 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/a2a3fdc49068424da43992a2b4b6475a is 50, key is test_row_0/A:col10/1733399778979/Put/seqid=0 2024-12-05T11:56:19,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741865_1041 (size=12409) 2024-12-05T11:56:19,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741864_1040 (size=12409) 2024-12-05T11:56:19,293 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/be69616aa13a487daecd3697973abc1d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/be69616aa13a487daecd3697973abc1d 2024-12-05T11:56:19,307 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/B of 41e385893d26f932b2fcba15c0b7f150 into be69616aa13a487daecd3697973abc1d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:19,307 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:19,307 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/B, priority=13, startTime=1733399779217; duration=0sec 2024-12-05T11:56:19,307 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:19,307 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:B 2024-12-05T11:56:19,307 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:19,311 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:19,311 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/C is initiating minor compaction (all files) 2024-12-05T11:56:19,311 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/C in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:19,311 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/af5d094374d64f85a6669bce8c8ce29f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/6991b238a930405bad403f2541b08749, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e08a989c584d4333b06a69ff39a3bc89] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=35.5 K 2024-12-05T11:56:19,312 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting af5d094374d64f85a6669bce8c8ce29f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399777564 2024-12-05T11:56:19,313 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6991b238a930405bad403f2541b08749, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733399778284 2024-12-05T11:56:19,315 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting e08a989c584d4333b06a69ff39a3bc89, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399778644 2024-12-05T11:56:19,337 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#C#compaction#29 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:19,338 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/a6e57132ef0f40eda068ec6b2c2ec001 is 50, key is test_row_0/C:col10/1733399778979/Put/seqid=0 2024-12-05T11:56:19,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741866_1042 (size=12409) 2024-12-05T11:56:19,364 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/a6e57132ef0f40eda068ec6b2c2ec001 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a6e57132ef0f40eda068ec6b2c2ec001 2024-12-05T11:56:19,380 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/C of 41e385893d26f932b2fcba15c0b7f150 into a6e57132ef0f40eda068ec6b2c2ec001(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:19,380 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:19,380 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/C, priority=13, startTime=1733399779218; duration=0sec 2024-12-05T11:56:19,381 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:19,381 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:C 2024-12-05T11:56:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:19,382 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:19,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T11:56:19,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/6af9b1f02af5400fa698a357492a9689 is 50, key is test_row_0/A:col10/1733399779053/Put/seqid=0 2024-12-05T11:56:19,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399839402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399839402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399839404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741867_1043 (size=14541) 2024-12-05T11:56:19,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399839518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399839519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399839519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,684 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/a2a3fdc49068424da43992a2b4b6475a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a2a3fdc49068424da43992a2b4b6475a 2024-12-05T11:56:19,697 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/A of 41e385893d26f932b2fcba15c0b7f150 into a2a3fdc49068424da43992a2b4b6475a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:19,697 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:19,697 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/A, priority=13, startTime=1733399779217; duration=0sec 2024-12-05T11:56:19,698 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:19,698 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:A 2024-12-05T11:56:19,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399839727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399839728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399839728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399839728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,734 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in 41e385893d26f932b2fcba15c0b7f150 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399839738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:19,741 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in 41e385893d26f932b2fcba15c0b7f150 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:19,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/6af9b1f02af5400fa698a357492a9689 2024-12-05T11:56:19,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/a2482dec0205476da9a98aff8a3d7d6a is 50, key is test_row_0/B:col10/1733399779053/Put/seqid=0 2024-12-05T11:56:19,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741868_1044 (size=12151) 2024-12-05T11:56:20,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399840038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:20,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399840038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:20,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399840038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:56:20,188 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:20,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-05T11:56:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T11:56:20,192 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:20,195 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:20,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:20,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/a2482dec0205476da9a98aff8a3d7d6a 2024-12-05T11:56:20,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/882ff189accc4258bc57ad2a9038c900 is 50, key is test_row_0/C:col10/1733399779053/Put/seqid=0 2024-12-05T11:56:20,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741869_1045 (size=12151) 2024-12-05T11:56:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T11:56:20,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T11:56:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:20,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:20,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:20,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T11:56:20,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:20,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:20,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T11:56:20,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:20,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:20,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399840548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:20,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399840548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:20,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399840548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:20,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T11:56:20,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:20,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:20,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:20,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:20,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:20,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/882ff189accc4258bc57ad2a9038c900 2024-12-05T11:56:20,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/6af9b1f02af5400fa698a357492a9689 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/6af9b1f02af5400fa698a357492a9689 2024-12-05T11:56:20,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/6af9b1f02af5400fa698a357492a9689, entries=200, sequenceid=161, filesize=14.2 K 2024-12-05T11:56:20,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/a2482dec0205476da9a98aff8a3d7d6a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/a2482dec0205476da9a98aff8a3d7d6a 2024-12-05T11:56:20,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/a2482dec0205476da9a98aff8a3d7d6a, entries=150, sequenceid=161, filesize=11.9 K 2024-12-05T11:56:20,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/882ff189accc4258bc57ad2a9038c900 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/882ff189accc4258bc57ad2a9038c900 2024-12-05T11:56:20,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/882ff189accc4258bc57ad2a9038c900, entries=150, sequenceid=161, filesize=11.9 K 2024-12-05T11:56:20,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 41e385893d26f932b2fcba15c0b7f150 in 1345ms, sequenceid=161, compaction requested=false 2024-12-05T11:56:20,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:20,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T11:56:20,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:20,816 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T11:56:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T11:56:20,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/df856310edc445e48e7db2be7d187d78 is 50, key is test_row_0/A:col10/1733399779400/Put/seqid=0 2024-12-05T11:56:20,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741870_1046 (size=12151) 2024-12-05T11:56:20,852 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/df856310edc445e48e7db2be7d187d78 2024-12-05T11:56:20,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/0f009afc958e4981b191bf980e6bbc93 is 50, key is test_row_0/B:col10/1733399779400/Put/seqid=0 2024-12-05T11:56:20,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741871_1047 (size=12151) 2024-12-05T11:56:20,888 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/0f009afc958e4981b191bf980e6bbc93 2024-12-05T11:56:20,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/f2dd0f5123e440248e1646a8ca15b26d is 50, key is test_row_0/C:col10/1733399779400/Put/seqid=0 2024-12-05T11:56:20,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741872_1048 (size=12151) 2024-12-05T11:56:20,926 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/f2dd0f5123e440248e1646a8ca15b26d 2024-12-05T11:56:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/df856310edc445e48e7db2be7d187d78 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/df856310edc445e48e7db2be7d187d78 2024-12-05T11:56:20,949 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/df856310edc445e48e7db2be7d187d78, entries=150, sequenceid=173, filesize=11.9 K 2024-12-05T11:56:20,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/0f009afc958e4981b191bf980e6bbc93 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/0f009afc958e4981b191bf980e6bbc93 2024-12-05T11:56:20,965 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/0f009afc958e4981b191bf980e6bbc93, entries=150, sequenceid=173, filesize=11.9 K 2024-12-05T11:56:20,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/f2dd0f5123e440248e1646a8ca15b26d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/f2dd0f5123e440248e1646a8ca15b26d 2024-12-05T11:56:20,984 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/f2dd0f5123e440248e1646a8ca15b26d, entries=150, sequenceid=173, filesize=11.9 K 2024-12-05T11:56:20,986 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 41e385893d26f932b2fcba15c0b7f150 in 171ms, sequenceid=173, compaction requested=true 2024-12-05T11:56:20,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:20,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:20,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-05T11:56:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-05T11:56:20,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T11:56:20,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 794 msec 2024-12-05T11:56:20,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 806 msec 2024-12-05T11:56:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T11:56:21,328 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:21,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-05T11:56:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T11:56:21,338 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:21,341 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:21,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T11:56:21,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T11:56:21,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:21,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:21,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:21,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-05T11:56:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-05T11:56:21,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-05T11:56:21,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 159 msec 2024-12-05T11:56:21,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 196 msec 2024-12-05T11:56:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:21,576 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:21,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T11:56:21,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/4e206cb8c5124c9c867304b08303329d is 50, key is test_row_0/A:col10/1733399781571/Put/seqid=0 2024-12-05T11:56:21,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741873_1049 (size=12151) 2024-12-05T11:56:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T11:56:21,658 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:21,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-05T11:56:21,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399841653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:56:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:21,663 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:21,664 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:21,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:21,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399841663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399841667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:21,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399841738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:21,740 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in 41e385893d26f932b2fcba15c0b7f150 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:21,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399841757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:21,761 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in 41e385893d26f932b2fcba15c0b7f150 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:56:21,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399841768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:21,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399841778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:21,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399841778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:21,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T11:56:21,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:21,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:21,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:21,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:21,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:21,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T11:56:21,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:21,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:21,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:21,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:21,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:56:21,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399841978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:21,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399841988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399841988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:22,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/4e206cb8c5124c9c867304b08303329d 2024-12-05T11:56:22,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/6414cbcd310e4985ae395208a6c399be is 50, key is test_row_0/B:col10/1733399781571/Put/seqid=0 2024-12-05T11:56:22,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741874_1050 (size=12151) 2024-12-05T11:56:22,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/6414cbcd310e4985ae395208a6c399be 2024-12-05T11:56:22,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/d5867cc8804c43d6b21d3e61111e426e is 50, key is test_row_0/C:col10/1733399781571/Put/seqid=0 2024-12-05T11:56:22,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741875_1051 (size=12151) 2024-12-05T11:56:22,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/d5867cc8804c43d6b21d3e61111e426e 2024-12-05T11:56:22,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/4e206cb8c5124c9c867304b08303329d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/4e206cb8c5124c9c867304b08303329d 2024-12-05T11:56:22,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/4e206cb8c5124c9c867304b08303329d, entries=150, sequenceid=185, filesize=11.9 K 2024-12-05T11:56:22,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/6414cbcd310e4985ae395208a6c399be as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/6414cbcd310e4985ae395208a6c399be 2024-12-05T11:56:22,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T11:56:22,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:22,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:22,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:22,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:22,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:22,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/6414cbcd310e4985ae395208a6c399be, entries=150, sequenceid=185, filesize=11.9 K 2024-12-05T11:56:22,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/d5867cc8804c43d6b21d3e61111e426e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/d5867cc8804c43d6b21d3e61111e426e 2024-12-05T11:56:22,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/d5867cc8804c43d6b21d3e61111e426e, entries=150, sequenceid=185, filesize=11.9 K 2024-12-05T11:56:22,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 41e385893d26f932b2fcba15c0b7f150 in 581ms, sequenceid=185, compaction requested=true 2024-12-05T11:56:22,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:22,158 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:56:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:22,158 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:56:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:22,159 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51252 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:56:22,160 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/A is initiating minor compaction (all files) 2024-12-05T11:56:22,160 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/A in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:22,160 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:56:22,160 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/B is initiating minor compaction (all files) 2024-12-05T11:56:22,160 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a2a3fdc49068424da43992a2b4b6475a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/6af9b1f02af5400fa698a357492a9689, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/df856310edc445e48e7db2be7d187d78, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/4e206cb8c5124c9c867304b08303329d] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=50.1 K 2024-12-05T11:56:22,160 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/B in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:22,160 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/be69616aa13a487daecd3697973abc1d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/a2482dec0205476da9a98aff8a3d7d6a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/0f009afc958e4981b191bf980e6bbc93, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/6414cbcd310e4985ae395208a6c399be] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=47.7 K 2024-12-05T11:56:22,161 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2a3fdc49068424da43992a2b4b6475a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399778644 2024-12-05T11:56:22,162 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6af9b1f02af5400fa698a357492a9689, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733399779043 2024-12-05T11:56:22,162 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting be69616aa13a487daecd3697973abc1d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399778644 2024-12-05T11:56:22,163 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting df856310edc445e48e7db2be7d187d78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733399779399 2024-12-05T11:56:22,164 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting a2482dec0205476da9a98aff8a3d7d6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733399779050 2024-12-05T11:56:22,164 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e206cb8c5124c9c867304b08303329d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733399781569 2024-12-05T11:56:22,165 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f009afc958e4981b191bf980e6bbc93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733399779399 2024-12-05T11:56:22,166 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6414cbcd310e4985ae395208a6c399be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733399781569 2024-12-05T11:56:22,182 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:22,183 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/933a33e8f2ce411d904d5aa6f0ee4775 is 50, key is test_row_0/A:col10/1733399781571/Put/seqid=0 2024-12-05T11:56:22,193 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:22,194 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/3cb4ba5018e14ded95f2add9317da829 is 50, key is test_row_0/B:col10/1733399781571/Put/seqid=0 2024-12-05T11:56:22,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741876_1052 (size=12595) 2024-12-05T11:56:22,213 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/933a33e8f2ce411d904d5aa6f0ee4775 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/933a33e8f2ce411d904d5aa6f0ee4775 2024-12-05T11:56:22,224 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/A of 41e385893d26f932b2fcba15c0b7f150 into 933a33e8f2ce411d904d5aa6f0ee4775(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:22,224 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:22,224 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/A, priority=12, startTime=1733399782157; duration=0sec 2024-12-05T11:56:22,225 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:22,225 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:A 2024-12-05T11:56:22,225 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:56:22,231 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:56:22,231 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/C is initiating minor compaction (all files) 2024-12-05T11:56:22,232 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/C in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:22,232 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a6e57132ef0f40eda068ec6b2c2ec001, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/882ff189accc4258bc57ad2a9038c900, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/f2dd0f5123e440248e1646a8ca15b26d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/d5867cc8804c43d6b21d3e61111e426e] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=47.7 K 2024-12-05T11:56:22,233 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6e57132ef0f40eda068ec6b2c2ec001, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399778644 2024-12-05T11:56:22,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741877_1053 (size=12595) 2024-12-05T11:56:22,235 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 882ff189accc4258bc57ad2a9038c900, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733399779050 2024-12-05T11:56:22,236 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2dd0f5123e440248e1646a8ca15b26d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733399779399 2024-12-05T11:56:22,236 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5867cc8804c43d6b21d3e61111e426e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733399781569 2024-12-05T11:56:22,244 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/3cb4ba5018e14ded95f2add9317da829 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/3cb4ba5018e14ded95f2add9317da829 2024-12-05T11:56:22,258 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/B of 41e385893d26f932b2fcba15c0b7f150 into 3cb4ba5018e14ded95f2add9317da829(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:22,258 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:22,258 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/B, priority=12, startTime=1733399782158; duration=0sec 2024-12-05T11:56:22,258 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:22,258 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:B 2024-12-05T11:56:22,269 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#C#compaction#41 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:22,270 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/a7cf3e7d100144448ce40903561a663a is 50, key is test_row_0/C:col10/1733399781571/Put/seqid=0 2024-12-05T11:56:22,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741878_1054 (size=12595) 2024-12-05T11:56:22,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T11:56:22,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:22,286 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-05T11:56:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:56:22,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:22,297 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/a7cf3e7d100144448ce40903561a663a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a7cf3e7d100144448ce40903561a663a 2024-12-05T11:56:22,311 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/C of 41e385893d26f932b2fcba15c0b7f150 into a7cf3e7d100144448ce40903561a663a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:22,311 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:22,311 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/C, priority=12, startTime=1733399782158; duration=0sec 2024-12-05T11:56:22,311 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:22,311 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:C 2024-12-05T11:56:22,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399842306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:22,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399842309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/a1ef1e6b73ab4d1084c1da523f39e3f3 is 50, key is test_row_0/A:col10/1733399781659/Put/seqid=0 2024-12-05T11:56:22,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399842312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:22,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741879_1055 (size=12151) 2024-12-05T11:56:22,333 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/a1ef1e6b73ab4d1084c1da523f39e3f3 2024-12-05T11:56:22,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/ad37f2e7d8f64d8db91319e0969c9dca is 50, key is test_row_0/B:col10/1733399781659/Put/seqid=0 2024-12-05T11:56:22,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741880_1056 (size=12151) 2024-12-05T11:56:22,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:22,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399842418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:22,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399842418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399842418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:22,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:22,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399842628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399842628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399842628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:22,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:22,783 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/ad37f2e7d8f64d8db91319e0969c9dca 2024-12-05T11:56:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:56:22,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/db33ceaee58f4a0c96c57b3f41d81504 is 50, key is test_row_0/C:col10/1733399781659/Put/seqid=0 2024-12-05T11:56:22,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741881_1057 (size=12151) 2024-12-05T11:56:22,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:22,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399842938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399842938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399842938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:23,206 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/db33ceaee58f4a0c96c57b3f41d81504 2024-12-05T11:56:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/a1ef1e6b73ab4d1084c1da523f39e3f3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a1ef1e6b73ab4d1084c1da523f39e3f3 2024-12-05T11:56:23,226 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a1ef1e6b73ab4d1084c1da523f39e3f3, entries=150, sequenceid=213, filesize=11.9 K 2024-12-05T11:56:23,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/ad37f2e7d8f64d8db91319e0969c9dca as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ad37f2e7d8f64d8db91319e0969c9dca 2024-12-05T11:56:23,243 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ad37f2e7d8f64d8db91319e0969c9dca, entries=150, sequenceid=213, filesize=11.9 K 2024-12-05T11:56:23,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/db33ceaee58f4a0c96c57b3f41d81504 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/db33ceaee58f4a0c96c57b3f41d81504 2024-12-05T11:56:23,252 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/db33ceaee58f4a0c96c57b3f41d81504, entries=150, sequenceid=213, filesize=11.9 K 2024-12-05T11:56:23,258 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 41e385893d26f932b2fcba15c0b7f150 in 972ms, sequenceid=213, compaction requested=false 2024-12-05T11:56:23,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:23,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:23,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-05T11:56:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-05T11:56:23,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-05T11:56:23,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5970 sec 2024-12-05T11:56:23,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6070 sec 2024-12-05T11:56:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:23,450 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:23,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T11:56:23,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/1da123d70b154bb8bb1744657e690169 is 50, key is test_row_0/A:col10/1733399782302/Put/seqid=0 2024-12-05T11:56:23,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741882_1058 (size=12151) 2024-12-05T11:56:23,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399843493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399843493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:23,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399843495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:23,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399843608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399843608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399843608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:56:23,808 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:23,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-05T11:56:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T11:56:23,810 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:23,811 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:23,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:23,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399843818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:23,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399843818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:23,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399843818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:23,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/1da123d70b154bb8bb1744657e690169 2024-12-05T11:56:23,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/29b0d5ab8f1c41978b9295fc0adcc6d1 is 50, key is test_row_0/B:col10/1733399782302/Put/seqid=0 2024-12-05T11:56:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T11:56:23,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741883_1059 (size=12151) 2024-12-05T11:56:23,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T11:56:23,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:23,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:23,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:23,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:23,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T11:56:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T11:56:24,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399844128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:24,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399844128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:24,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399844128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:24,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T11:56:24,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:24,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/29b0d5ab8f1c41978b9295fc0adcc6d1 2024-12-05T11:56:24,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/08375032446445449e7e628a7d43052e is 50, key is test_row_0/C:col10/1733399782302/Put/seqid=0 2024-12-05T11:56:24,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741884_1060 (size=12151) 2024-12-05T11:56:24,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T11:56:24,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:24,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T11:56:24,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T11:56:24,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:24,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399844638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:24,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399844638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:24,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399844638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:24,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T11:56:24,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:24,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:24,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/08375032446445449e7e628a7d43052e 2024-12-05T11:56:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/1da123d70b154bb8bb1744657e690169 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/1da123d70b154bb8bb1744657e690169 2024-12-05T11:56:24,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/1da123d70b154bb8bb1744657e690169, entries=150, sequenceid=225, filesize=11.9 K 2024-12-05T11:56:24,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/29b0d5ab8f1c41978b9295fc0adcc6d1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/29b0d5ab8f1c41978b9295fc0adcc6d1 2024-12-05T11:56:24,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/29b0d5ab8f1c41978b9295fc0adcc6d1, entries=150, sequenceid=225, filesize=11.9 K 2024-12-05T11:56:24,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/08375032446445449e7e628a7d43052e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/08375032446445449e7e628a7d43052e 2024-12-05T11:56:24,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/08375032446445449e7e628a7d43052e, entries=150, sequenceid=225, filesize=11.9 K 2024-12-05T11:56:24,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 41e385893d26f932b2fcba15c0b7f150 in 1326ms, sequenceid=225, compaction requested=true 2024-12-05T11:56:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:24,777 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:24,777 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:24,779 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:24,779 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/B is initiating minor compaction (all files) 2024-12-05T11:56:24,779 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/B in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,779 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/3cb4ba5018e14ded95f2add9317da829, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ad37f2e7d8f64d8db91319e0969c9dca, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/29b0d5ab8f1c41978b9295fc0adcc6d1] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=36.0 K 2024-12-05T11:56:24,780 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:24,780 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cb4ba5018e14ded95f2add9317da829, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733399781569 2024-12-05T11:56:24,780 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/A is initiating minor compaction (all files) 2024-12-05T11:56:24,780 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/A in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,780 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/933a33e8f2ce411d904d5aa6f0ee4775, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a1ef1e6b73ab4d1084c1da523f39e3f3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/1da123d70b154bb8bb1744657e690169] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=36.0 K 2024-12-05T11:56:24,781 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ad37f2e7d8f64d8db91319e0969c9dca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733399781650 2024-12-05T11:56:24,781 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 933a33e8f2ce411d904d5aa6f0ee4775, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733399781569 2024-12-05T11:56:24,781 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 29b0d5ab8f1c41978b9295fc0adcc6d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733399782295 2024-12-05T11:56:24,782 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1ef1e6b73ab4d1084c1da523f39e3f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733399781650 2024-12-05T11:56:24,783 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1da123d70b154bb8bb1744657e690169, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733399782295 2024-12-05T11:56:24,797 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#B#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:24,797 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:24,797 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/dc96fbde35d54e2a9e26f213a2a9134f is 50, key is test_row_0/B:col10/1733399782302/Put/seqid=0 2024-12-05T11:56:24,798 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/f460d083a3e34574876a74f9ab95c65d is 50, key is test_row_0/A:col10/1733399782302/Put/seqid=0 2024-12-05T11:56:24,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741885_1061 (size=12697) 2024-12-05T11:56:24,827 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/dc96fbde35d54e2a9e26f213a2a9134f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/dc96fbde35d54e2a9e26f213a2a9134f 2024-12-05T11:56:24,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741886_1062 (size=12697) 2024-12-05T11:56:24,837 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/B of 41e385893d26f932b2fcba15c0b7f150 into dc96fbde35d54e2a9e26f213a2a9134f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:24,837 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:24,837 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/B, priority=13, startTime=1733399784777; duration=0sec 2024-12-05T11:56:24,838 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:24,838 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:B 2024-12-05T11:56:24,838 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:24,842 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:24,842 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/C is initiating minor compaction (all files) 2024-12-05T11:56:24,843 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/C in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,843 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a7cf3e7d100144448ce40903561a663a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/db33ceaee58f4a0c96c57b3f41d81504, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/08375032446445449e7e628a7d43052e] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=36.0 K 2024-12-05T11:56:24,844 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting a7cf3e7d100144448ce40903561a663a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733399781569 2024-12-05T11:56:24,848 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/f460d083a3e34574876a74f9ab95c65d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/f460d083a3e34574876a74f9ab95c65d 2024-12-05T11:56:24,848 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting db33ceaee58f4a0c96c57b3f41d81504, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733399781650 2024-12-05T11:56:24,849 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 08375032446445449e7e628a7d43052e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733399782295 2024-12-05T11:56:24,860 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/A of 41e385893d26f932b2fcba15c0b7f150 into f460d083a3e34574876a74f9ab95c65d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:24,860 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:24,860 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/A, priority=13, startTime=1733399784777; duration=0sec 2024-12-05T11:56:24,861 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:24,861 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:A 2024-12-05T11:56:24,864 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#C#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:24,865 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/895495768fe54a7e8351040add0a697a is 50, key is test_row_0/C:col10/1733399782302/Put/seqid=0 2024-12-05T11:56:24,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741887_1063 (size=12697) 2024-12-05T11:56:24,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T11:56:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:24,892 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T11:56:24,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/b2764102530a4a07893aadd04740e756 is 50, key is test_row_0/A:col10/1733399783491/Put/seqid=0 2024-12-05T11:56:24,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741888_1064 (size=12151) 2024-12-05T11:56:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T11:56:25,284 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/895495768fe54a7e8351040add0a697a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/895495768fe54a7e8351040add0a697a 2024-12-05T11:56:25,293 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/C of 41e385893d26f932b2fcba15c0b7f150 into 895495768fe54a7e8351040add0a697a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:25,293 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:25,293 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/C, priority=13, startTime=1733399784777; duration=0sec 2024-12-05T11:56:25,293 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:25,293 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:C 2024-12-05T11:56:25,311 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/b2764102530a4a07893aadd04740e756 2024-12-05T11:56:25,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/ba675a18a1164530ae16ac67ad53b487 is 50, key is test_row_0/B:col10/1733399783491/Put/seqid=0 2024-12-05T11:56:25,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741889_1065 (size=12151) 2024-12-05T11:56:25,333 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/ba675a18a1164530ae16ac67ad53b487 2024-12-05T11:56:25,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/bffcc26541aa44c4a1068ac0f87ad2d6 is 50, key is test_row_0/C:col10/1733399783491/Put/seqid=0 2024-12-05T11:56:25,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741890_1066 (size=12151) 2024-12-05T11:56:25,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:25,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399845661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:25,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399845663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:25,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399845664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:25,752 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/bffcc26541aa44c4a1068ac0f87ad2d6 2024-12-05T11:56:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/b2764102530a4a07893aadd04740e756 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/b2764102530a4a07893aadd04740e756 2024-12-05T11:56:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399845757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:25,763 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in 41e385893d26f932b2fcba15c0b7f150 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:25,767 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/b2764102530a4a07893aadd04740e756, entries=150, sequenceid=252, filesize=11.9 K 2024-12-05T11:56:25,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/ba675a18a1164530ae16ac67ad53b487 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ba675a18a1164530ae16ac67ad53b487 2024-12-05T11:56:25,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399845768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399845769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:25,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399845779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:25,779 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ba675a18a1164530ae16ac67ad53b487, entries=150, sequenceid=252, filesize=11.9 K 2024-12-05T11:56:25,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399845779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:25,780 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in 41e385893d26f932b2fcba15c0b7f150 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:25,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/bffcc26541aa44c4a1068ac0f87ad2d6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/bffcc26541aa44c4a1068ac0f87ad2d6 2024-12-05T11:56:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:25,789 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/bffcc26541aa44c4a1068ac0f87ad2d6, entries=150, sequenceid=252, filesize=11.9 K 2024-12-05T11:56:25,791 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 41e385893d26f932b2fcba15c0b7f150 in 899ms, sequenceid=252, compaction requested=false 2024-12-05T11:56:25,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:25,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:25,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-05T11:56:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-05T11:56:25,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-05T11:56:25,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9820 sec 2024-12-05T11:56:25,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.9900 sec 2024-12-05T11:56:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T11:56:25,958 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:25,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-05T11:56:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T11:56:25,961 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:25,961 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:25,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:25,980 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:25,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T11:56:25,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/8136405a1c744c6c8161c4f517d15a84 is 50, key is test_row_0/A:col10/1733399785660/Put/seqid=0 2024-12-05T11:56:26,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741891_1067 (size=14741) 2024-12-05T11:56:26,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/8136405a1c744c6c8161c4f517d15a84 2024-12-05T11:56:26,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/f2459c39e0f745a4af753829a4ff237d is 50, key is test_row_0/B:col10/1733399785660/Put/seqid=0 2024-12-05T11:56:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399846018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399846020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741892_1068 (size=12301) 2024-12-05T11:56:26,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399846023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T11:56:26,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T11:56:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:26,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:26,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399846128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399846128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399846138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T11:56:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:26,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T11:56:26,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399846338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399846338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399846347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T11:56:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:26,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:26,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/f2459c39e0f745a4af753829a4ff237d 2024-12-05T11:56:26,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/e86272253f434f7d8b64ec90457f77cf is 50, key is test_row_0/C:col10/1733399785660/Put/seqid=0 2024-12-05T11:56:26,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741893_1069 (size=12301) 2024-12-05T11:56:26,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/e86272253f434f7d8b64ec90457f77cf 2024-12-05T11:56:26,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/8136405a1c744c6c8161c4f517d15a84 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8136405a1c744c6c8161c4f517d15a84 2024-12-05T11:56:26,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8136405a1c744c6c8161c4f517d15a84, entries=200, sequenceid=266, filesize=14.4 K 2024-12-05T11:56:26,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/f2459c39e0f745a4af753829a4ff237d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f2459c39e0f745a4af753829a4ff237d 2024-12-05T11:56:26,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f2459c39e0f745a4af753829a4ff237d, entries=150, sequenceid=266, filesize=12.0 K 2024-12-05T11:56:26,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/e86272253f434f7d8b64ec90457f77cf as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e86272253f434f7d8b64ec90457f77cf 2024-12-05T11:56:26,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e86272253f434f7d8b64ec90457f77cf, entries=150, sequenceid=266, filesize=12.0 K 2024-12-05T11:56:26,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 41e385893d26f932b2fcba15c0b7f150 in 505ms, sequenceid=266, compaction requested=true 2024-12-05T11:56:26,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:26,485 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:26,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:26,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:26,486 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:26,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:26,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:26,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:26,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:26,487 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:26,487 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/A is initiating minor compaction (all files) 2024-12-05T11:56:26,487 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/A in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:26,487 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/f460d083a3e34574876a74f9ab95c65d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/b2764102530a4a07893aadd04740e756, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8136405a1c744c6c8161c4f517d15a84] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=38.7 K 2024-12-05T11:56:26,488 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:26,488 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/B is initiating minor compaction (all files) 2024-12-05T11:56:26,488 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/B in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:26,488 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/dc96fbde35d54e2a9e26f213a2a9134f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ba675a18a1164530ae16ac67ad53b487, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f2459c39e0f745a4af753829a4ff237d] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=36.3 K 2024-12-05T11:56:26,488 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting dc96fbde35d54e2a9e26f213a2a9134f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733399782295 2024-12-05T11:56:26,489 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f460d083a3e34574876a74f9ab95c65d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733399782295 2024-12-05T11:56:26,489 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ba675a18a1164530ae16ac67ad53b487, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733399783491 2024-12-05T11:56:26,489 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2764102530a4a07893aadd04740e756, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733399783491 2024-12-05T11:56:26,490 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting f2459c39e0f745a4af753829a4ff237d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733399785660 2024-12-05T11:56:26,490 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8136405a1c744c6c8161c4f517d15a84, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733399785660 2024-12-05T11:56:26,512 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#B#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:26,513 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/2511ba5ce9454eebad9d33c839fa81e0 is 50, key is test_row_0/B:col10/1733399785660/Put/seqid=0 2024-12-05T11:56:26,515 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#A#compaction#58 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:26,516 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/abfe05618ef34326b44d5ca4c8de1b23 is 50, key is test_row_0/A:col10/1733399785660/Put/seqid=0 2024-12-05T11:56:26,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741894_1070 (size=12949) 2024-12-05T11:56:26,534 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/2511ba5ce9454eebad9d33c839fa81e0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/2511ba5ce9454eebad9d33c839fa81e0 2024-12-05T11:56:26,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741895_1071 (size=12949) 2024-12-05T11:56:26,543 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/B of 41e385893d26f932b2fcba15c0b7f150 into 2511ba5ce9454eebad9d33c839fa81e0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:26,543 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:26,543 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/B, priority=13, startTime=1733399786486; duration=0sec 2024-12-05T11:56:26,543 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:26,544 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:B 2024-12-05T11:56:26,544 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:26,548 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/abfe05618ef34326b44d5ca4c8de1b23 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/abfe05618ef34326b44d5ca4c8de1b23 2024-12-05T11:56:26,552 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:26,552 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/C is initiating minor compaction (all files) 2024-12-05T11:56:26,552 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/C in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:26,552 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/895495768fe54a7e8351040add0a697a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/bffcc26541aa44c4a1068ac0f87ad2d6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e86272253f434f7d8b64ec90457f77cf] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=36.3 K 2024-12-05T11:56:26,554 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 895495768fe54a7e8351040add0a697a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733399782295 2024-12-05T11:56:26,555 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting bffcc26541aa44c4a1068ac0f87ad2d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733399783491 2024-12-05T11:56:26,555 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting e86272253f434f7d8b64ec90457f77cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733399785660 2024-12-05T11:56:26,559 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/A of 41e385893d26f932b2fcba15c0b7f150 into abfe05618ef34326b44d5ca4c8de1b23(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:26,559 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:26,559 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/A, priority=13, startTime=1733399786485; duration=0sec 2024-12-05T11:56:26,559 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:26,559 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:A 2024-12-05T11:56:26,565 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:26,565 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/a940eb1f544540f6a6abdcbaff87707b is 50, key is test_row_0/C:col10/1733399785660/Put/seqid=0 2024-12-05T11:56:26,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741896_1072 (size=12949) 2024-12-05T11:56:26,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T11:56:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:26,578 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T11:56:26,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/35311c18e4f5444aa6f0223132717d87 is 50, key is test_row_0/A:col10/1733399786018/Put/seqid=0 2024-12-05T11:56:26,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741897_1073 (size=12301) 2024-12-05T11:56:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T11:56:26,589 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/35311c18e4f5444aa6f0223132717d87 2024-12-05T11:56:26,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/0b664f0514d14be6b0a0ba75171ab0ba is 50, key is test_row_0/B:col10/1733399786018/Put/seqid=0 2024-12-05T11:56:26,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741898_1074 (size=12301) 2024-12-05T11:56:26,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:26,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399846666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399846667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399846667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399846778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399846778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399846778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,981 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/a940eb1f544540f6a6abdcbaff87707b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a940eb1f544540f6a6abdcbaff87707b 2024-12-05T11:56:26,989 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/C of 41e385893d26f932b2fcba15c0b7f150 into a940eb1f544540f6a6abdcbaff87707b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:26,989 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:26,989 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/C, priority=13, startTime=1733399786486; duration=0sec 2024-12-05T11:56:26,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399846988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,989 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:26,989 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:C 2024-12-05T11:56:26,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399846988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:26,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399846988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:27,009 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/0b664f0514d14be6b0a0ba75171ab0ba 2024-12-05T11:56:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/64557fd9206c4a558c39c6febdcd80bd is 50, key is test_row_0/C:col10/1733399786018/Put/seqid=0 2024-12-05T11:56:27,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741899_1075 (size=12301) 2024-12-05T11:56:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T11:56:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399847298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399847298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:27,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399847298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:27,446 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/64557fd9206c4a558c39c6febdcd80bd 2024-12-05T11:56:27,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/35311c18e4f5444aa6f0223132717d87 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/35311c18e4f5444aa6f0223132717d87 2024-12-05T11:56:27,460 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/35311c18e4f5444aa6f0223132717d87, entries=150, sequenceid=292, filesize=12.0 K 2024-12-05T11:56:27,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/0b664f0514d14be6b0a0ba75171ab0ba as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/0b664f0514d14be6b0a0ba75171ab0ba 2024-12-05T11:56:27,469 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/0b664f0514d14be6b0a0ba75171ab0ba, entries=150, sequenceid=292, filesize=12.0 K 2024-12-05T11:56:27,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/64557fd9206c4a558c39c6febdcd80bd as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/64557fd9206c4a558c39c6febdcd80bd 2024-12-05T11:56:27,480 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/64557fd9206c4a558c39c6febdcd80bd, entries=150, sequenceid=292, filesize=12.0 K 2024-12-05T11:56:27,481 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 41e385893d26f932b2fcba15c0b7f150 in 903ms, sequenceid=292, compaction requested=false 2024-12-05T11:56:27,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:27,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:27,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-05T11:56:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-05T11:56:27,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-05T11:56:27,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5220 sec 2024-12-05T11:56:27,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5320 sec 2024-12-05T11:56:27,809 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:27,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-05T11:56:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:27,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/7bfd42af3c4340a7b09a580b1ca2242c is 50, key is test_row_0/A:col10/1733399787808/Put/seqid=0 2024-12-05T11:56:27,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741900_1076 (size=12301) 2024-12-05T11:56:27,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399847846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:27,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:27,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399847846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399847847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:27,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399847958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:27,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399847958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:27,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:27,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399847958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T11:56:28,108 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:28,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-05T11:56:28,111 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T11:56:28,112 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:28,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:28,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399848168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:28,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399848168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:28,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399848168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T11:56:28,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/7bfd42af3c4340a7b09a580b1ca2242c 2024-12-05T11:56:28,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/ed6c4056a1894f359b5b22acd79702ad is 50, key is test_row_0/B:col10/1733399787808/Put/seqid=0 2024-12-05T11:56:28,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741901_1077 (size=12301) 2024-12-05T11:56:28,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T11:56:28,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:28,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:28,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:28,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T11:56:28,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:28,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:28,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:28,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T11:56:28,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399848478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399848478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399848478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:28,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T11:56:28,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:28,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:28,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:28,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/ed6c4056a1894f359b5b22acd79702ad 2024-12-05T11:56:28,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/25df00a5a50b4feab506511c31533afd is 50, key is test_row_0/C:col10/1733399787808/Put/seqid=0 2024-12-05T11:56:28,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741902_1078 (size=12301) 2024-12-05T11:56:28,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T11:56:28,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:28,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:28,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:28,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T11:56:28,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T11:56:28,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:28,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:28,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:28,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:28,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399848987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399848988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:28,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399848988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:29,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T11:56:29,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:29,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:29,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:29,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:29,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:29,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/25df00a5a50b4feab506511c31533afd 2024-12-05T11:56:29,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/7bfd42af3c4340a7b09a580b1ca2242c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/7bfd42af3c4340a7b09a580b1ca2242c 2024-12-05T11:56:29,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/7bfd42af3c4340a7b09a580b1ca2242c, entries=150, sequenceid=309, filesize=12.0 K 2024-12-05T11:56:29,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/ed6c4056a1894f359b5b22acd79702ad as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ed6c4056a1894f359b5b22acd79702ad 2024-12-05T11:56:29,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ed6c4056a1894f359b5b22acd79702ad, entries=150, sequenceid=309, filesize=12.0 K 2024-12-05T11:56:29,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/25df00a5a50b4feab506511c31533afd as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/25df00a5a50b4feab506511c31533afd 2024-12-05T11:56:29,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/25df00a5a50b4feab506511c31533afd, entries=150, sequenceid=309, filesize=12.0 K 2024-12-05T11:56:29,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 41e385893d26f932b2fcba15c0b7f150 in 1278ms, sequenceid=309, compaction requested=true 2024-12-05T11:56:29,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:29,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:29,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:29,087 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:29,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:29,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:29,087 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:29,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:29,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:29,089 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:29,089 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:29,089 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/B is initiating minor compaction (all files) 2024-12-05T11:56:29,089 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/A is initiating minor compaction (all files) 2024-12-05T11:56:29,089 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/B in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:29,089 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/A in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:29,089 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/abfe05618ef34326b44d5ca4c8de1b23, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/35311c18e4f5444aa6f0223132717d87, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/7bfd42af3c4340a7b09a580b1ca2242c] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=36.7 K 2024-12-05T11:56:29,089 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/2511ba5ce9454eebad9d33c839fa81e0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/0b664f0514d14be6b0a0ba75171ab0ba, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ed6c4056a1894f359b5b22acd79702ad] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=36.7 K 2024-12-05T11:56:29,090 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2511ba5ce9454eebad9d33c839fa81e0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733399785660 2024-12-05T11:56:29,090 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting abfe05618ef34326b44d5ca4c8de1b23, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733399785660 2024-12-05T11:56:29,090 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b664f0514d14be6b0a0ba75171ab0ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733399786004 2024-12-05T11:56:29,090 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35311c18e4f5444aa6f0223132717d87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733399786004 2024-12-05T11:56:29,091 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bfd42af3c4340a7b09a580b1ca2242c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733399786662 2024-12-05T11:56:29,091 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ed6c4056a1894f359b5b22acd79702ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733399786662 2024-12-05T11:56:29,100 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#A#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:29,101 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/88e34637c22e49c08ccb6220b11487bc is 50, key is test_row_0/A:col10/1733399787808/Put/seqid=0 2024-12-05T11:56:29,102 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#B#compaction#67 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:29,103 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/cd2129f24dd64b04968b2bfc7eb83109 is 50, key is test_row_0/B:col10/1733399787808/Put/seqid=0 2024-12-05T11:56:29,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741903_1079 (size=13051) 2024-12-05T11:56:29,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741904_1080 (size=13051) 2024-12-05T11:56:29,121 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/cd2129f24dd64b04968b2bfc7eb83109 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/cd2129f24dd64b04968b2bfc7eb83109 2024-12-05T11:56:29,127 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/B of 41e385893d26f932b2fcba15c0b7f150 into cd2129f24dd64b04968b2bfc7eb83109(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:29,128 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:29,128 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/B, priority=13, startTime=1733399789087; duration=0sec 2024-12-05T11:56:29,128 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:29,128 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:B 2024-12-05T11:56:29,128 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:29,129 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:29,129 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/C is initiating minor compaction (all files) 2024-12-05T11:56:29,129 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/C in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:29,129 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a940eb1f544540f6a6abdcbaff87707b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/64557fd9206c4a558c39c6febdcd80bd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/25df00a5a50b4feab506511c31533afd] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=36.7 K 2024-12-05T11:56:29,130 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting a940eb1f544540f6a6abdcbaff87707b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733399785660 2024-12-05T11:56:29,130 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 64557fd9206c4a558c39c6febdcd80bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733399786004 2024-12-05T11:56:29,131 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 25df00a5a50b4feab506511c31533afd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733399786662 2024-12-05T11:56:29,137 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:29,138 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/0e7b8e51379f44eb924f286a956c3b55 is 50, key is test_row_0/C:col10/1733399787808/Put/seqid=0 2024-12-05T11:56:29,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741905_1081 (size=13051) 2024-12-05T11:56:29,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T11:56:29,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:29,185 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T11:56:29,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/e7fd624ab7e2485cb5533d54dfde7221 is 50, key is test_row_0/A:col10/1733399787840/Put/seqid=0 2024-12-05T11:56:29,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741906_1082 (size=12301) 2024-12-05T11:56:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T11:56:29,519 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/88e34637c22e49c08ccb6220b11487bc as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/88e34637c22e49c08ccb6220b11487bc 2024-12-05T11:56:29,526 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/A of 41e385893d26f932b2fcba15c0b7f150 into 88e34637c22e49c08ccb6220b11487bc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:29,526 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:29,526 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/A, priority=13, startTime=1733399789087; duration=0sec 2024-12-05T11:56:29,527 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:29,527 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:A 2024-12-05T11:56:29,551 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/0e7b8e51379f44eb924f286a956c3b55 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/0e7b8e51379f44eb924f286a956c3b55 2024-12-05T11:56:29,560 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/C of 41e385893d26f932b2fcba15c0b7f150 into 0e7b8e51379f44eb924f286a956c3b55(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:29,560 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:29,560 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/C, priority=13, startTime=1733399789087; duration=0sec 2024-12-05T11:56:29,560 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:29,560 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:C 2024-12-05T11:56:29,596 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/e7fd624ab7e2485cb5533d54dfde7221 2024-12-05T11:56:29,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/d8a197c509c54a3682a04181bf2a6430 is 50, key is test_row_0/B:col10/1733399787840/Put/seqid=0 2024-12-05T11:56:29,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741907_1083 (size=12301) 2024-12-05T11:56:29,624 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/d8a197c509c54a3682a04181bf2a6430 2024-12-05T11:56:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/49e65b3660da48979918c6e992f5a0b1 is 50, key is test_row_0/C:col10/1733399787840/Put/seqid=0 2024-12-05T11:56:29,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741908_1084 (size=12301) 2024-12-05T11:56:29,646 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/49e65b3660da48979918c6e992f5a0b1 2024-12-05T11:56:29,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/e7fd624ab7e2485cb5533d54dfde7221 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/e7fd624ab7e2485cb5533d54dfde7221 2024-12-05T11:56:29,668 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/e7fd624ab7e2485cb5533d54dfde7221, entries=150, sequenceid=331, filesize=12.0 K 2024-12-05T11:56:29,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/d8a197c509c54a3682a04181bf2a6430 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d8a197c509c54a3682a04181bf2a6430 2024-12-05T11:56:29,677 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d8a197c509c54a3682a04181bf2a6430, entries=150, sequenceid=331, filesize=12.0 K 2024-12-05T11:56:29,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/49e65b3660da48979918c6e992f5a0b1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/49e65b3660da48979918c6e992f5a0b1 2024-12-05T11:56:29,686 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/49e65b3660da48979918c6e992f5a0b1, entries=150, sequenceid=331, filesize=12.0 K 2024-12-05T11:56:29,687 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 41e385893d26f932b2fcba15c0b7f150 in 502ms, sequenceid=331, compaction requested=false 2024-12-05T11:56:29,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-05T11:56:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-05T11:56:29,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-05T11:56:29,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5770 sec 2024-12-05T11:56:29,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5840 sec 2024-12-05T11:56:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:30,004 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:30,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T11:56:30,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/f77cb20a22a54818a3af2c7b62213a7a is 50, key is test_row_0/A:col10/1733399790004/Put/seqid=0 2024-12-05T11:56:30,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741909_1085 (size=14741) 2024-12-05T11:56:30,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399850029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:30,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399850030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399850032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:30,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:30,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399850138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399850138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399850138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T11:56:30,258 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:30,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-05T11:56:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:56:30,260 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:30,261 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:30,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:30,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399850348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:30,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399850348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:30,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399850348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:56:30,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:56:30,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:30,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:30,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:30,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:30,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:30,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/f77cb20a22a54818a3af2c7b62213a7a 2024-12-05T11:56:30,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/5450a04483da4d3aa696a0b03cc21cfe is 50, key is test_row_0/B:col10/1733399790004/Put/seqid=0 2024-12-05T11:56:30,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741910_1086 (size=12301) 2024-12-05T11:56:30,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:56:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:30,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:56:30,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399850658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:30,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399850658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:30,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399850658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:30,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:30,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:30,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:56:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:30,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:30,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/5450a04483da4d3aa696a0b03cc21cfe 2024-12-05T11:56:30,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/0898a695f47d4dbf9321620ffdab46b1 is 50, key is test_row_0/C:col10/1733399790004/Put/seqid=0 2024-12-05T11:56:30,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741911_1087 (size=12301) 2024-12-05T11:56:30,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:56:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:30,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:30,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:56:31,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:56:31,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:31,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:31,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:31,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:31,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:31,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399851167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:31,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399851167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:31,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399851168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:31,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:56:31,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:31,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:31,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:31,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:31,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:31,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/0898a695f47d4dbf9321620ffdab46b1 2024-12-05T11:56:31,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/f77cb20a22a54818a3af2c7b62213a7a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/f77cb20a22a54818a3af2c7b62213a7a 2024-12-05T11:56:31,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/f77cb20a22a54818a3af2c7b62213a7a, entries=200, sequenceid=346, filesize=14.4 K 2024-12-05T11:56:31,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/5450a04483da4d3aa696a0b03cc21cfe as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/5450a04483da4d3aa696a0b03cc21cfe 2024-12-05T11:56:31,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/5450a04483da4d3aa696a0b03cc21cfe, entries=150, sequenceid=346, filesize=12.0 K 2024-12-05T11:56:31,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/0898a695f47d4dbf9321620ffdab46b1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/0898a695f47d4dbf9321620ffdab46b1 2024-12-05T11:56:31,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/0898a695f47d4dbf9321620ffdab46b1, entries=150, sequenceid=346, filesize=12.0 K 2024-12-05T11:56:31,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 41e385893d26f932b2fcba15c0b7f150 in 1283ms, sequenceid=346, compaction requested=true 2024-12-05T11:56:31,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:31,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:31,288 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:31,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:31,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:31,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:31,288 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:31,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:31,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:31,289 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:31,289 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/A is initiating minor compaction (all files) 2024-12-05T11:56:31,289 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:31,289 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/A in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:31,289 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/B is initiating minor compaction (all files) 2024-12-05T11:56:31,289 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/B in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:31,289 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/88e34637c22e49c08ccb6220b11487bc, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/e7fd624ab7e2485cb5533d54dfde7221, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/f77cb20a22a54818a3af2c7b62213a7a] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=39.2 K 2024-12-05T11:56:31,290 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/cd2129f24dd64b04968b2bfc7eb83109, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d8a197c509c54a3682a04181bf2a6430, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/5450a04483da4d3aa696a0b03cc21cfe] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=36.8 K 2024-12-05T11:56:31,290 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88e34637c22e49c08ccb6220b11487bc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733399786662 2024-12-05T11:56:31,290 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2129f24dd64b04968b2bfc7eb83109, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733399786662 2024-12-05T11:56:31,290 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d8a197c509c54a3682a04181bf2a6430, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733399787826 2024-12-05T11:56:31,290 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7fd624ab7e2485cb5533d54dfde7221, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733399787826 2024-12-05T11:56:31,291 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5450a04483da4d3aa696a0b03cc21cfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733399790003 2024-12-05T11:56:31,291 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f77cb20a22a54818a3af2c7b62213a7a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733399790000 2024-12-05T11:56:31,297 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:31,298 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/c03e4b8a49f34b7b99a7ba6b958dd7ef is 50, key is test_row_0/A:col10/1733399790004/Put/seqid=0 2024-12-05T11:56:31,299 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#B#compaction#76 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:31,300 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/5321df1c2e7140eab0dda68aff38b35c is 50, key is test_row_0/B:col10/1733399790004/Put/seqid=0 2024-12-05T11:56:31,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741912_1088 (size=13153) 2024-12-05T11:56:31,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741913_1089 (size=13153) 2024-12-05T11:56:31,316 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/5321df1c2e7140eab0dda68aff38b35c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/5321df1c2e7140eab0dda68aff38b35c 2024-12-05T11:56:31,322 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/B of 41e385893d26f932b2fcba15c0b7f150 into 5321df1c2e7140eab0dda68aff38b35c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:31,322 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:31,322 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/B, priority=13, startTime=1733399791288; duration=0sec 2024-12-05T11:56:31,323 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:31,323 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:B 2024-12-05T11:56:31,323 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:31,324 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:31,324 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/C is initiating minor compaction (all files) 2024-12-05T11:56:31,324 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/C in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:31,324 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/0e7b8e51379f44eb924f286a956c3b55, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/49e65b3660da48979918c6e992f5a0b1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/0898a695f47d4dbf9321620ffdab46b1] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=36.8 K 2024-12-05T11:56:31,325 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e7b8e51379f44eb924f286a956c3b55, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733399786662 2024-12-05T11:56:31,325 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 49e65b3660da48979918c6e992f5a0b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733399787826 2024-12-05T11:56:31,325 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0898a695f47d4dbf9321620ffdab46b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733399790003 2024-12-05T11:56:31,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:56:31,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:31,333 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T11:56:31,333 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#C#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:31,334 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/63e7e2dd5b3947d9a31b8450a171bb7d is 50, key is test_row_0/C:col10/1733399790004/Put/seqid=0 2024-12-05T11:56:31,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/5dc3b9f83e974058a742ee6f7a8059ad is 50, key is test_row_0/A:col10/1733399790024/Put/seqid=0 2024-12-05T11:56:31,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741914_1090 (size=13153) 2024-12-05T11:56:31,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741915_1091 (size=12301) 2024-12-05T11:56:31,344 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/5dc3b9f83e974058a742ee6f7a8059ad 2024-12-05T11:56:31,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/53d47acdbcfa439aaf6dba6e8d134b0e is 50, key is test_row_0/B:col10/1733399790024/Put/seqid=0 2024-12-05T11:56:31,356 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/63e7e2dd5b3947d9a31b8450a171bb7d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/63e7e2dd5b3947d9a31b8450a171bb7d 2024-12-05T11:56:31,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741916_1092 (size=12301) 2024-12-05T11:56:31,363 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/C of 41e385893d26f932b2fcba15c0b7f150 into 63e7e2dd5b3947d9a31b8450a171bb7d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:31,363 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:31,363 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/C, priority=13, startTime=1733399791288; duration=0sec 2024-12-05T11:56:31,363 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:31,363 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:C 2024-12-05T11:56:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:56:31,710 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/c03e4b8a49f34b7b99a7ba6b958dd7ef as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/c03e4b8a49f34b7b99a7ba6b958dd7ef 2024-12-05T11:56:31,716 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/A of 41e385893d26f932b2fcba15c0b7f150 into c03e4b8a49f34b7b99a7ba6b958dd7ef(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:31,716 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:31,716 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/A, priority=13, startTime=1733399791288; duration=0sec 2024-12-05T11:56:31,716 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:31,716 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:A 2024-12-05T11:56:31,759 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/53d47acdbcfa439aaf6dba6e8d134b0e 2024-12-05T11:56:31,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/796b76a32dc545aca7befcf3fcb0c827 is 50, key is test_row_0/C:col10/1733399790024/Put/seqid=0 2024-12-05T11:56:31,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741917_1093 (size=12301) 2024-12-05T11:56:32,176 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/796b76a32dc545aca7befcf3fcb0c827 2024-12-05T11:56:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:32,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:32,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/5dc3b9f83e974058a742ee6f7a8059ad as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5dc3b9f83e974058a742ee6f7a8059ad 2024-12-05T11:56:32,190 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5dc3b9f83e974058a742ee6f7a8059ad, entries=150, sequenceid=370, filesize=12.0 K 2024-12-05T11:56:32,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/53d47acdbcfa439aaf6dba6e8d134b0e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/53d47acdbcfa439aaf6dba6e8d134b0e 2024-12-05T11:56:32,199 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/53d47acdbcfa439aaf6dba6e8d134b0e, entries=150, sequenceid=370, filesize=12.0 K 2024-12-05T11:56:32,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/796b76a32dc545aca7befcf3fcb0c827 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/796b76a32dc545aca7befcf3fcb0c827 2024-12-05T11:56:32,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399852198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399852199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399852199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,207 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/796b76a32dc545aca7befcf3fcb0c827, entries=150, sequenceid=370, filesize=12.0 K 2024-12-05T11:56:32,208 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 41e385893d26f932b2fcba15c0b7f150 in 875ms, sequenceid=370, compaction requested=false 2024-12-05T11:56:32,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:32,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:32,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-05T11:56:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-05T11:56:32,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-05T11:56:32,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9490 sec 2024-12-05T11:56:32,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.9550 sec 2024-12-05T11:56:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:32,310 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:32,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-05T11:56:32,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/a3e90092342c483dbd6be890330594c2 is 50, key is test_row_0/A:col10/1733399792309/Put/seqid=0 2024-12-05T11:56:32,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741918_1094 (size=14741) 2024-12-05T11:56:32,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/a3e90092342c483dbd6be890330594c2 2024-12-05T11:56:32,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/c2ddf0e333004da98f8ab8665721b9b9 is 50, key is test_row_0/B:col10/1733399792309/Put/seqid=0 2024-12-05T11:56:32,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741919_1095 (size=12301) 2024-12-05T11:56:32,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399852333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399852335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399852338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:56:32,408 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:32,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-05T11:56:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T11:56:32,411 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:32,412 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:32,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:32,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399852448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399852448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399852448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T11:56:32,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-05T11:56:32,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:32,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:32,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:32,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:32,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:32,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399852658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399852658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399852658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-05T11:56:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:32,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T11:56:32,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/c2ddf0e333004da98f8ab8665721b9b9 2024-12-05T11:56:32,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/5918f0921c0c4e7091199a5e03c151c0 is 50, key is test_row_0/C:col10/1733399792309/Put/seqid=0 2024-12-05T11:56:32,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741920_1096 (size=12301) 2024-12-05T11:56:32,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-05T11:56:32,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:32,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:32,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:32,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:32,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:32,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399852968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399852968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399852968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:33,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-05T11:56:33,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:33,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:33,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:33,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:33,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T11:56:33,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/5918f0921c0c4e7091199a5e03c151c0 2024-12-05T11:56:33,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/a3e90092342c483dbd6be890330594c2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a3e90092342c483dbd6be890330594c2 2024-12-05T11:56:33,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a3e90092342c483dbd6be890330594c2, entries=200, sequenceid=387, filesize=14.4 K 2024-12-05T11:56:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/c2ddf0e333004da98f8ab8665721b9b9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/c2ddf0e333004da98f8ab8665721b9b9 2024-12-05T11:56:33,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/c2ddf0e333004da98f8ab8665721b9b9, entries=150, sequenceid=387, filesize=12.0 K 2024-12-05T11:56:33,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/5918f0921c0c4e7091199a5e03c151c0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/5918f0921c0c4e7091199a5e03c151c0 2024-12-05T11:56:33,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/5918f0921c0c4e7091199a5e03c151c0, entries=150, sequenceid=387, filesize=12.0 K 2024-12-05T11:56:33,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 41e385893d26f932b2fcba15c0b7f150 in 861ms, sequenceid=387, compaction requested=true 2024-12-05T11:56:33,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:33,171 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:33,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:33,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:33,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:33,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:33,171 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:33,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:33,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:33,172 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:33,172 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:33,172 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/A is initiating minor compaction (all files) 2024-12-05T11:56:33,172 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/B is initiating minor compaction (all files) 2024-12-05T11:56:33,172 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/A in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:33,172 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/B in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:33,173 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/c03e4b8a49f34b7b99a7ba6b958dd7ef, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5dc3b9f83e974058a742ee6f7a8059ad, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a3e90092342c483dbd6be890330594c2] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=39.3 K 2024-12-05T11:56:33,173 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/5321df1c2e7140eab0dda68aff38b35c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/53d47acdbcfa439aaf6dba6e8d134b0e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/c2ddf0e333004da98f8ab8665721b9b9] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=36.9 K 2024-12-05T11:56:33,173 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5321df1c2e7140eab0dda68aff38b35c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733399790003 2024-12-05T11:56:33,173 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c03e4b8a49f34b7b99a7ba6b958dd7ef, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733399790003 2024-12-05T11:56:33,173 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d47acdbcfa439aaf6dba6e8d134b0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733399790024 2024-12-05T11:56:33,173 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dc3b9f83e974058a742ee6f7a8059ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733399790024 2024-12-05T11:56:33,174 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c2ddf0e333004da98f8ab8665721b9b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733399792193 2024-12-05T11:56:33,174 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3e90092342c483dbd6be890330594c2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733399792193 2024-12-05T11:56:33,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-05T11:56:33,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:33,177 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T11:56:33,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/5a488be729c34979b692a947a04b73f2 is 50, key is test_row_0/A:col10/1733399792336/Put/seqid=0 2024-12-05T11:56:33,192 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:33,193 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/c5780e5d71b74bce843e2b411c1fbf4f is 50, key is test_row_0/B:col10/1733399792309/Put/seqid=0 2024-12-05T11:56:33,194 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#A#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:33,195 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/09baad3f38844276bcdf425bf9d9c92d is 50, key is test_row_0/A:col10/1733399792309/Put/seqid=0 2024-12-05T11:56:33,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741923_1099 (size=13255) 2024-12-05T11:56:33,213 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/09baad3f38844276bcdf425bf9d9c92d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/09baad3f38844276bcdf425bf9d9c92d 2024-12-05T11:56:33,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741921_1097 (size=12301) 2024-12-05T11:56:33,215 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/5a488be729c34979b692a947a04b73f2 2024-12-05T11:56:33,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741922_1098 (size=13255) 2024-12-05T11:56:33,223 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/A of 41e385893d26f932b2fcba15c0b7f150 into 09baad3f38844276bcdf425bf9d9c92d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:33,223 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:33,223 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/A, priority=13, startTime=1733399793171; duration=0sec 2024-12-05T11:56:33,223 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:33,223 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:A 2024-12-05T11:56:33,223 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:33,225 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:33,225 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/C is initiating minor compaction (all files) 2024-12-05T11:56:33,225 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/C in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:33,226 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/63e7e2dd5b3947d9a31b8450a171bb7d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/796b76a32dc545aca7befcf3fcb0c827, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/5918f0921c0c4e7091199a5e03c151c0] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=36.9 K 2024-12-05T11:56:33,226 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63e7e2dd5b3947d9a31b8450a171bb7d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733399790003 2024-12-05T11:56:33,227 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/c5780e5d71b74bce843e2b411c1fbf4f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/c5780e5d71b74bce843e2b411c1fbf4f 2024-12-05T11:56:33,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/d0deb9e499124d3bbe8550890b916ccf is 50, key is test_row_0/B:col10/1733399792336/Put/seqid=0 2024-12-05T11:56:33,230 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 796b76a32dc545aca7befcf3fcb0c827, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733399790024 2024-12-05T11:56:33,230 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5918f0921c0c4e7091199a5e03c151c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733399792193 2024-12-05T11:56:33,235 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/B of 41e385893d26f932b2fcba15c0b7f150 into c5780e5d71b74bce843e2b411c1fbf4f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:33,235 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:33,235 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/B, priority=13, startTime=1733399793171; duration=0sec 2024-12-05T11:56:33,235 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:33,235 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:B 2024-12-05T11:56:33,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741924_1100 (size=12301) 2024-12-05T11:56:33,237 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/d0deb9e499124d3bbe8550890b916ccf 2024-12-05T11:56:33,242 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#C#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:33,242 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/9bcb0c3bfc724d1c8a7db6188695d393 is 50, key is test_row_0/C:col10/1733399792309/Put/seqid=0 2024-12-05T11:56:33,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/ee98d8374c8d4fe5af2e097a174c5750 is 50, key is test_row_0/C:col10/1733399792336/Put/seqid=0 2024-12-05T11:56:33,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741925_1101 (size=13255) 2024-12-05T11:56:33,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741926_1102 (size=12301) 2024-12-05T11:56:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:33,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:33,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399853488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:33,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399853489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:33,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399853492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:33,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T11:56:33,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399853598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:33,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:33,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399853600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:33,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399853601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:33,683 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/ee98d8374c8d4fe5af2e097a174c5750 2024-12-05T11:56:33,689 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/9bcb0c3bfc724d1c8a7db6188695d393 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/9bcb0c3bfc724d1c8a7db6188695d393 2024-12-05T11:56:33,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/5a488be729c34979b692a947a04b73f2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5a488be729c34979b692a947a04b73f2 2024-12-05T11:56:33,694 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5a488be729c34979b692a947a04b73f2, entries=150, sequenceid=408, filesize=12.0 K 2024-12-05T11:56:33,695 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/C of 41e385893d26f932b2fcba15c0b7f150 into 9bcb0c3bfc724d1c8a7db6188695d393(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:33,695 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:33,695 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/C, priority=13, startTime=1733399793171; duration=0sec 2024-12-05T11:56:33,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/d0deb9e499124d3bbe8550890b916ccf as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d0deb9e499124d3bbe8550890b916ccf 2024-12-05T11:56:33,696 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:33,696 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:C 2024-12-05T11:56:33,701 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d0deb9e499124d3bbe8550890b916ccf, entries=150, sequenceid=408, filesize=12.0 K 2024-12-05T11:56:33,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/ee98d8374c8d4fe5af2e097a174c5750 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/ee98d8374c8d4fe5af2e097a174c5750 2024-12-05T11:56:33,707 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/ee98d8374c8d4fe5af2e097a174c5750, entries=150, sequenceid=408, filesize=12.0 K 2024-12-05T11:56:33,709 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 41e385893d26f932b2fcba15c0b7f150 in 532ms, sequenceid=408, compaction requested=false 2024-12-05T11:56:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-05T11:56:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-05T11:56:33,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-05T11:56:33,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2990 sec 2024-12-05T11:56:33,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.3040 sec 2024-12-05T11:56:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:33,810 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:33,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-05T11:56:33,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/0341a01852204e78b01f18a47f956e40 is 50, key is test_row_0/A:col10/1733399793810/Put/seqid=0 2024-12-05T11:56:33,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741927_1103 (size=12301) 2024-12-05T11:56:33,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:33,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399853843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399853844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:33,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399853850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:33,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:33,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399853957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:33,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399853957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:33,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399853957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399854168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399854168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399854168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/0341a01852204e78b01f18a47f956e40 2024-12-05T11:56:34,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/21bd7cfb95464840bc18f465d3e5339a is 50, key is test_row_0/B:col10/1733399793810/Put/seqid=0 2024-12-05T11:56:34,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741928_1104 (size=12301) 2024-12-05T11:56:34,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/21bd7cfb95464840bc18f465d3e5339a 2024-12-05T11:56:34,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/d09a1ec349014e0a9494ff4754b65af6 is 50, key is test_row_0/C:col10/1733399793810/Put/seqid=0 2024-12-05T11:56:34,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741929_1105 (size=12301) 2024-12-05T11:56:34,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/d09a1ec349014e0a9494ff4754b65af6 2024-12-05T11:56:34,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/0341a01852204e78b01f18a47f956e40 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/0341a01852204e78b01f18a47f956e40 2024-12-05T11:56:34,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/0341a01852204e78b01f18a47f956e40, entries=150, sequenceid=429, filesize=12.0 K 2024-12-05T11:56:34,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/21bd7cfb95464840bc18f465d3e5339a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/21bd7cfb95464840bc18f465d3e5339a 2024-12-05T11:56:34,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/21bd7cfb95464840bc18f465d3e5339a, entries=150, sequenceid=429, filesize=12.0 K 2024-12-05T11:56:34,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/d09a1ec349014e0a9494ff4754b65af6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/d09a1ec349014e0a9494ff4754b65af6 2024-12-05T11:56:34,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/d09a1ec349014e0a9494ff4754b65af6, entries=150, sequenceid=429, filesize=12.0 K 2024-12-05T11:56:34,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 41e385893d26f932b2fcba15c0b7f150 in 500ms, sequenceid=429, compaction requested=true 2024-12-05T11:56:34,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:34,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:34,311 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:34,311 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:34,312 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:34,312 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:34,312 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/A is initiating minor compaction (all files) 2024-12-05T11:56:34,312 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/B is initiating minor compaction (all files) 2024-12-05T11:56:34,312 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/B in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:34,312 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/A in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:34,312 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/c5780e5d71b74bce843e2b411c1fbf4f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d0deb9e499124d3bbe8550890b916ccf, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/21bd7cfb95464840bc18f465d3e5339a] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=37.0 K 2024-12-05T11:56:34,312 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/09baad3f38844276bcdf425bf9d9c92d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5a488be729c34979b692a947a04b73f2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/0341a01852204e78b01f18a47f956e40] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=37.0 K 2024-12-05T11:56:34,313 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c5780e5d71b74bce843e2b411c1fbf4f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733399792193 2024-12-05T11:56:34,313 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09baad3f38844276bcdf425bf9d9c92d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733399792193 2024-12-05T11:56:34,313 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a488be729c34979b692a947a04b73f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733399792332 2024-12-05T11:56:34,313 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d0deb9e499124d3bbe8550890b916ccf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733399792332 2024-12-05T11:56:34,313 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0341a01852204e78b01f18a47f956e40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733399793483 2024-12-05T11:56:34,314 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 21bd7cfb95464840bc18f465d3e5339a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733399793483 2024-12-05T11:56:34,321 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#A#compaction#93 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:34,322 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/d67b36150cf04237b5246b2ec67922ce is 50, key is test_row_0/A:col10/1733399793810/Put/seqid=0 2024-12-05T11:56:34,324 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:34,324 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/37e24a56c29b42a18751062a6f3b9dff is 50, key is test_row_0/B:col10/1733399793810/Put/seqid=0 2024-12-05T11:56:34,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741931_1107 (size=13357) 2024-12-05T11:56:34,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741930_1106 (size=13357) 2024-12-05T11:56:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:34,480 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:34,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-05T11:56:34,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/06dc47f98c9144c6be1974d9c1e0339b is 50, key is test_row_0/A:col10/1733399793843/Put/seqid=0 2024-12-05T11:56:34,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741932_1108 (size=17181) 2024-12-05T11:56:34,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399854499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399854502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:34,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:34,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399854503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T11:56:34,558 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:34,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-05T11:56:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T11:56:34,560 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:34,560 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:34,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:34,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399854608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:34,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399854617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:34,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399854617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T11:56:34,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:34,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:34,740 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/37e24a56c29b42a18751062a6f3b9dff as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/37e24a56c29b42a18751062a6f3b9dff 2024-12-05T11:56:34,746 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/B of 41e385893d26f932b2fcba15c0b7f150 into 37e24a56c29b42a18751062a6f3b9dff(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:34,746 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:34,747 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/B, priority=13, startTime=1733399794311; duration=0sec 2024-12-05T11:56:34,747 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:34,747 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:B 2024-12-05T11:56:34,747 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:34,749 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:34,749 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/C is initiating minor compaction (all files) 2024-12-05T11:56:34,749 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/C in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:34,750 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/9bcb0c3bfc724d1c8a7db6188695d393, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/ee98d8374c8d4fe5af2e097a174c5750, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/d09a1ec349014e0a9494ff4754b65af6] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=37.0 K 2024-12-05T11:56:34,750 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/d67b36150cf04237b5246b2ec67922ce as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/d67b36150cf04237b5246b2ec67922ce 2024-12-05T11:56:34,750 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bcb0c3bfc724d1c8a7db6188695d393, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733399792193 2024-12-05T11:56:34,751 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ee98d8374c8d4fe5af2e097a174c5750, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733399792332 2024-12-05T11:56:34,751 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d09a1ec349014e0a9494ff4754b65af6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733399793483 2024-12-05T11:56:34,756 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/A of 41e385893d26f932b2fcba15c0b7f150 into d67b36150cf04237b5246b2ec67922ce(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:34,756 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:34,756 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/A, priority=13, startTime=1733399794310; duration=0sec 2024-12-05T11:56:34,756 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:34,756 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:A 2024-12-05T11:56:34,758 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#C#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:34,759 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/ab251302cd1f4f82ae3cad9c55540ef8 is 50, key is test_row_0/C:col10/1733399793810/Put/seqid=0 2024-12-05T11:56:34,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741933_1109 (size=13357) 2024-12-05T11:56:34,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399854817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399854828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399854828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:34,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:34,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:34,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:34,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:34,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:34,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T11:56:34,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/06dc47f98c9144c6be1974d9c1e0339b 2024-12-05T11:56:34,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/43bd7b4d4c8945b9ad4739da380490fa is 50, key is test_row_0/B:col10/1733399793843/Put/seqid=0 2024-12-05T11:56:34,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741934_1110 (size=12301) 2024-12-05T11:56:35,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:35,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:35,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399855128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:35,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399855137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:35,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399855137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:35,171 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/ab251302cd1f4f82ae3cad9c55540ef8 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/ab251302cd1f4f82ae3cad9c55540ef8 2024-12-05T11:56:35,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:35,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:35,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,177 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/C of 41e385893d26f932b2fcba15c0b7f150 into ab251302cd1f4f82ae3cad9c55540ef8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:35,177 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:35,177 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/C, priority=13, startTime=1733399794311; duration=0sec 2024-12-05T11:56:35,177 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:35,178 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:C 2024-12-05T11:56:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T11:56:35,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/43bd7b4d4c8945b9ad4739da380490fa 2024-12-05T11:56:35,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/3f3297c10157460d9ae0e9c4c388c3fe is 50, key is test_row_0/C:col10/1733399793843/Put/seqid=0 2024-12-05T11:56:35,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741935_1111 (size=12301) 2024-12-05T11:56:35,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:35,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:35,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:35,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:35,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:35,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:35,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399855637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:35,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:35,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399855648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399855648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T11:56:35,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/3f3297c10157460d9ae0e9c4c388c3fe 2024-12-05T11:56:35,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/06dc47f98c9144c6be1974d9c1e0339b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/06dc47f98c9144c6be1974d9c1e0339b 2024-12-05T11:56:35,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/06dc47f98c9144c6be1974d9c1e0339b, entries=250, sequenceid=450, filesize=16.8 K 2024-12-05T11:56:35,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/43bd7b4d4c8945b9ad4739da380490fa as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/43bd7b4d4c8945b9ad4739da380490fa 2024-12-05T11:56:35,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/43bd7b4d4c8945b9ad4739da380490fa, entries=150, sequenceid=450, filesize=12.0 K 2024-12-05T11:56:35,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/3f3297c10157460d9ae0e9c4c388c3fe as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/3f3297c10157460d9ae0e9c4c388c3fe 2024-12-05T11:56:35,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/3f3297c10157460d9ae0e9c4c388c3fe, entries=150, sequenceid=450, filesize=12.0 K 2024-12-05T11:56:35,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 41e385893d26f932b2fcba15c0b7f150 in 1275ms, sequenceid=450, compaction requested=false 2024-12-05T11:56:35,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:35,780 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:35,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-05T11:56:35,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:35,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:35,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/7c6d4588d4114840bf7fb44a86ddaebb is 50, key is test_row_0/A:col10/1733399794502/Put/seqid=0 2024-12-05T11:56:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741936_1112 (size=14741) 2024-12-05T11:56:35,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399855831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:35,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399855831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:35,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:35,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:35,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:35,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399855937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:35,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399855938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:35,987 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-05T11:56:35,987 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-05T11:56:35,987 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:56:35,987 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:56:35,991 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:35,991 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:35,992 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:35,992 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:35,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:56:35,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:56:35,994 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-05T11:56:35,994 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:56:35,994 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:35,994 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:35,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:56:35,995 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-05T11:56:35,995 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:56:35,995 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:35,995 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:35,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:56:36,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:36,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:36,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:36,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:36,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:36,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399856149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399856149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:36,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:36,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/7c6d4588d4114840bf7fb44a86ddaebb 2024-12-05T11:56:36,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/52b6a6554ea9409ebc252a84de7140d9 is 50, key is test_row_0/B:col10/1733399794502/Put/seqid=0 2024-12-05T11:56:36,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741937_1113 (size=12301) 2024-12-05T11:56:36,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:36,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:36,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:36,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:36,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:36,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:36,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:36,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399856460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399856459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:36,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:36,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:36,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:36,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/52b6a6554ea9409ebc252a84de7140d9 2024-12-05T11:56:36,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/6f7511dc08364a7094b051afb47aff19 is 50, key is test_row_0/C:col10/1733399794502/Put/seqid=0 2024-12-05T11:56:36,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741938_1114 (size=12301) 2024-12-05T11:56:36,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1733399856647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:36,648 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_1' in 41e385893d26f932b2fcba15c0b7f150 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:36,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:36,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36230 deadline: 1733399856658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36258 deadline: 1733399856658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:36,659 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in 41e385893d26f932b2fcba15c0b7f150 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:36,659 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_1' in 41e385893d26f932b2fcba15c0b7f150 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T11:56:36,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:36,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:36,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:36,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:36,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:36,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:36,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:36,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:36,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:36,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:36,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:36,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1733399856968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:36,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36266 deadline: 1733399856968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:56:37,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:37,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:37,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:37,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:37,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:37,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:37,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/6f7511dc08364a7094b051afb47aff19 2024-12-05T11:56:37,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/7c6d4588d4114840bf7fb44a86ddaebb as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/7c6d4588d4114840bf7fb44a86ddaebb 2024-12-05T11:56:37,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/7c6d4588d4114840bf7fb44a86ddaebb, entries=200, sequenceid=469, filesize=14.4 K 2024-12-05T11:56:37,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/52b6a6554ea9409ebc252a84de7140d9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/52b6a6554ea9409ebc252a84de7140d9 2024-12-05T11:56:37,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/52b6a6554ea9409ebc252a84de7140d9, entries=150, sequenceid=469, filesize=12.0 K 2024-12-05T11:56:37,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/6f7511dc08364a7094b051afb47aff19 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/6f7511dc08364a7094b051afb47aff19 2024-12-05T11:56:37,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/6f7511dc08364a7094b051afb47aff19, entries=150, sequenceid=469, filesize=12.0 K 2024-12-05T11:56:37,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 41e385893d26f932b2fcba15c0b7f150 in 1290ms, sequenceid=469, compaction requested=true 2024-12-05T11:56:37,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:37,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:37,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:37,071 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:37,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:37,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:37,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41e385893d26f932b2fcba15c0b7f150:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:37,071 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:37,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:37,072 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:37,072 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:37,072 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/A is initiating minor compaction (all files) 2024-12-05T11:56:37,072 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/B is initiating minor compaction (all files) 2024-12-05T11:56:37,072 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/A in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:37,072 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/B in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:37,072 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/d67b36150cf04237b5246b2ec67922ce, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/06dc47f98c9144c6be1974d9c1e0339b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/7c6d4588d4114840bf7fb44a86ddaebb] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=44.2 K 2024-12-05T11:56:37,072 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/37e24a56c29b42a18751062a6f3b9dff, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/43bd7b4d4c8945b9ad4739da380490fa, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/52b6a6554ea9409ebc252a84de7140d9] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=37.1 K 2024-12-05T11:56:37,072 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d67b36150cf04237b5246b2ec67922ce, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733399793483 2024-12-05T11:56:37,072 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 37e24a56c29b42a18751062a6f3b9dff, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733399793483 2024-12-05T11:56:37,073 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06dc47f98c9144c6be1974d9c1e0339b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1733399793825 2024-12-05T11:56:37,073 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 43bd7b4d4c8945b9ad4739da380490fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1733399793843 2024-12-05T11:56:37,073 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6d4588d4114840bf7fb44a86ddaebb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733399794498 2024-12-05T11:56:37,073 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 52b6a6554ea9409ebc252a84de7140d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733399794501 2024-12-05T11:56:37,080 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#A#compaction#102 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:37,081 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/8c13571057dd457196bdde5ff98440f3 is 50, key is test_row_0/A:col10/1733399794502/Put/seqid=0 2024-12-05T11:56:37,081 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:37,082 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/227a736096484a8989cffad98ab87998 is 50, key is test_row_0/B:col10/1733399794502/Put/seqid=0 2024-12-05T11:56:37,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741939_1115 (size=13459) 2024-12-05T11:56:37,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741940_1116 (size=13459) 2024-12-05T11:56:37,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T11:56:37,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:37,183 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T11:56:37,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/ea68617f6b30474baf85a5f2a7e54fdd is 50, key is test_row_0/A:col10/1733399795824/Put/seqid=0 2024-12-05T11:56:37,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741941_1117 (size=12301) 2024-12-05T11:56:37,499 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/8c13571057dd457196bdde5ff98440f3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8c13571057dd457196bdde5ff98440f3 2024-12-05T11:56:37,499 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/227a736096484a8989cffad98ab87998 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/227a736096484a8989cffad98ab87998 2024-12-05T11:56:37,505 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/B of 41e385893d26f932b2fcba15c0b7f150 into 227a736096484a8989cffad98ab87998(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:37,505 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/A of 41e385893d26f932b2fcba15c0b7f150 into 8c13571057dd457196bdde5ff98440f3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:37,505 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:37,505 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:37,505 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/B, priority=13, startTime=1733399797071; duration=0sec 2024-12-05T11:56:37,505 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/A, priority=13, startTime=1733399797071; duration=0sec 2024-12-05T11:56:37,505 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:37,505 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:B 2024-12-05T11:56:37,505 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:37,505 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:37,505 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:A 2024-12-05T11:56:37,506 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:37,506 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 41e385893d26f932b2fcba15c0b7f150/C is initiating minor compaction (all files) 2024-12-05T11:56:37,506 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41e385893d26f932b2fcba15c0b7f150/C in TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:37,506 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/ab251302cd1f4f82ae3cad9c55540ef8, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/3f3297c10157460d9ae0e9c4c388c3fe, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/6f7511dc08364a7094b051afb47aff19] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp, totalSize=37.1 K 2024-12-05T11:56:37,507 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ab251302cd1f4f82ae3cad9c55540ef8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733399793483 2024-12-05T11:56:37,507 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f3297c10157460d9ae0e9c4c388c3fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1733399793843 2024-12-05T11:56:37,507 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f7511dc08364a7094b051afb47aff19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733399794501 2024-12-05T11:56:37,513 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41e385893d26f932b2fcba15c0b7f150#C#compaction#105 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:37,514 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/026aabca66ac4d5882f313f85449290d is 50, key is test_row_0/C:col10/1733399794502/Put/seqid=0 2024-12-05T11:56:37,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741942_1118 (size=13459) 2024-12-05T11:56:37,599 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/ea68617f6b30474baf85a5f2a7e54fdd 2024-12-05T11:56:37,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/d69de72416c841d89373f6d251281f9e is 50, key is test_row_0/B:col10/1733399795824/Put/seqid=0 2024-12-05T11:56:37,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741943_1119 (size=12301) 2024-12-05T11:56:37,924 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/026aabca66ac4d5882f313f85449290d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/026aabca66ac4d5882f313f85449290d 2024-12-05T11:56:37,929 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41e385893d26f932b2fcba15c0b7f150/C of 41e385893d26f932b2fcba15c0b7f150 into 026aabca66ac4d5882f313f85449290d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:37,929 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:37,929 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., storeName=41e385893d26f932b2fcba15c0b7f150/C, priority=13, startTime=1733399797071; duration=0sec 2024-12-05T11:56:37,930 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:37,930 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41e385893d26f932b2fcba15c0b7f150:C 2024-12-05T11:56:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:37,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. as already flushing 2024-12-05T11:56:37,981 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-05T11:56:37,981 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:56:37,981 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-05T11:56:37,981 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:37,981 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:56:37,981 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:37,981 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:37,981 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:37,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:56:37,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:56:38,018 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/d69de72416c841d89373f6d251281f9e 2024-12-05T11:56:38,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/f93caab3ec494b1e85669e1303f23c28 is 50, key is test_row_0/C:col10/1733399795824/Put/seqid=0 2024-12-05T11:56:38,034 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T11:56:38,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741944_1120 (size=12301) 2024-12-05T11:56:38,439 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/f93caab3ec494b1e85669e1303f23c28 2024-12-05T11:56:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/ea68617f6b30474baf85a5f2a7e54fdd as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/ea68617f6b30474baf85a5f2a7e54fdd 2024-12-05T11:56:38,456 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/ea68617f6b30474baf85a5f2a7e54fdd, entries=150, sequenceid=490, filesize=12.0 K 2024-12-05T11:56:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/d69de72416c841d89373f6d251281f9e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d69de72416c841d89373f6d251281f9e 2024-12-05T11:56:38,464 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d69de72416c841d89373f6d251281f9e, entries=150, sequenceid=490, filesize=12.0 K 2024-12-05T11:56:38,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/f93caab3ec494b1e85669e1303f23c28 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/f93caab3ec494b1e85669e1303f23c28 2024-12-05T11:56:38,470 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/f93caab3ec494b1e85669e1303f23c28, entries=150, sequenceid=490, filesize=12.0 K 2024-12-05T11:56:38,471 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=13.42 KB/13740 for 41e385893d26f932b2fcba15c0b7f150 in 1288ms, sequenceid=490, compaction requested=false 2024-12-05T11:56:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 41e385893d26f932b2fcba15c0b7f150: 2024-12-05T11:56:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-05T11:56:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-05T11:56:38,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-05T11:56:38,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9120 sec 2024-12-05T11:56:38,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 3.9170 sec 2024-12-05T11:56:38,660 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-05T11:56:38,660 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:56:38,660 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:38,660 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:38,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:56:38,671 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-05T11:56:38,671 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:56:38,671 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:38,671 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:38,671 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:56:38,679 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-05T11:56:38,679 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:56:38,679 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:38,679 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:38,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:56:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T11:56:38,728 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:38,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5253 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5316 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2374 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7118 rows 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2369 2024-12-05T11:56:38,729 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7103 rows 2024-12-05T11:56:38,729 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:56:38,730 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:56:38,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:38,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:38,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:56:38,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T11:56:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T11:56:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T11:56:38,743 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399798743"}]},"ts":"1733399798743"} 2024-12-05T11:56:38,745 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T11:56:38,745 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T11:56:38,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:56:38,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41e385893d26f932b2fcba15c0b7f150, UNASSIGN}] 2024-12-05T11:56:38,752 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41e385893d26f932b2fcba15c0b7f150, UNASSIGN 2024-12-05T11:56:38,754 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=41e385893d26f932b2fcba15c0b7f150, regionState=CLOSING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:56:38,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=41e385893d26f932b2fcba15c0b7f150, UNASSIGN because future has completed 2024-12-05T11:56:38,756 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:56:38,756 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:56:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T11:56:38,917 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:38,917 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:56:38,919 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 41e385893d26f932b2fcba15c0b7f150, disabling compactions & flushes 2024-12-05T11:56:38,919 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:38,919 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:38,919 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. after waiting 0 ms 2024-12-05T11:56:38,919 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:38,920 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 41e385893d26f932b2fcba15c0b7f150 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-05T11:56:38,927 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/ceb7fef73b204c37a0e139f6a87a4177 is 50, key is test_row_1/A:col10/1733399798678/Put/seqid=0 2024-12-05T11:56:38,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741945_1121 (size=9857) 2024-12-05T11:56:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T11:56:39,331 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/ceb7fef73b204c37a0e139f6a87a4177 2024-12-05T11:56:39,339 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/f3679883e62b435c884121bc6ea233d2 is 50, key is test_row_1/B:col10/1733399798678/Put/seqid=0 2024-12-05T11:56:39,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741946_1122 (size=9857) 2024-12-05T11:56:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T11:56:39,745 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/f3679883e62b435c884121bc6ea233d2 2024-12-05T11:56:39,752 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/60f6f1e5960e4e12923a88e37f2418a8 is 50, key is test_row_1/C:col10/1733399798678/Put/seqid=0 2024-12-05T11:56:39,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741947_1123 (size=9857) 2024-12-05T11:56:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T11:56:40,158 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/60f6f1e5960e4e12923a88e37f2418a8 2024-12-05T11:56:40,165 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/A/ceb7fef73b204c37a0e139f6a87a4177 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/ceb7fef73b204c37a0e139f6a87a4177 2024-12-05T11:56:40,169 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/ceb7fef73b204c37a0e139f6a87a4177, entries=100, sequenceid=501, filesize=9.6 K 2024-12-05T11:56:40,170 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/B/f3679883e62b435c884121bc6ea233d2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f3679883e62b435c884121bc6ea233d2 2024-12-05T11:56:40,174 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f3679883e62b435c884121bc6ea233d2, entries=100, sequenceid=501, filesize=9.6 K 2024-12-05T11:56:40,175 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/.tmp/C/60f6f1e5960e4e12923a88e37f2418a8 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/60f6f1e5960e4e12923a88e37f2418a8 2024-12-05T11:56:40,180 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/60f6f1e5960e4e12923a88e37f2418a8, entries=100, sequenceid=501, filesize=9.6 K 2024-12-05T11:56:40,180 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 41e385893d26f932b2fcba15c0b7f150 in 1261ms, sequenceid=501, compaction requested=true 2024-12-05T11:56:40,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/9c52c54b5eb1464196ddbcf5200ab053, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/638a7f1f8ab640118ad20abbdce366e5, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/42bef86dda1e4347ad59a7907143d74a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/2b2c56fc9d3a4c70bf235dca13671c5b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/1b1d932fb18c4dd8860bb3b91386c466, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/540fdfbcbecd432698cc96cb00866aa6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8118d40a82694c178f222bcc12d5d8c2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5924132f784f416fa59279e9114cfd70, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/4d60b24621cc458393523d972fa1e6af, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a2a3fdc49068424da43992a2b4b6475a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/6af9b1f02af5400fa698a357492a9689, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/df856310edc445e48e7db2be7d187d78, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/933a33e8f2ce411d904d5aa6f0ee4775, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/4e206cb8c5124c9c867304b08303329d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a1ef1e6b73ab4d1084c1da523f39e3f3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/f460d083a3e34574876a74f9ab95c65d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/1da123d70b154bb8bb1744657e690169, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/b2764102530a4a07893aadd04740e756, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8136405a1c744c6c8161c4f517d15a84, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/abfe05618ef34326b44d5ca4c8de1b23, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/35311c18e4f5444aa6f0223132717d87, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/88e34637c22e49c08ccb6220b11487bc, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/7bfd42af3c4340a7b09a580b1ca2242c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/e7fd624ab7e2485cb5533d54dfde7221, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/f77cb20a22a54818a3af2c7b62213a7a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/c03e4b8a49f34b7b99a7ba6b958dd7ef, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5dc3b9f83e974058a742ee6f7a8059ad, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a3e90092342c483dbd6be890330594c2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/09baad3f38844276bcdf425bf9d9c92d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5a488be729c34979b692a947a04b73f2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/d67b36150cf04237b5246b2ec67922ce, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/0341a01852204e78b01f18a47f956e40, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/06dc47f98c9144c6be1974d9c1e0339b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/7c6d4588d4114840bf7fb44a86ddaebb] to archive 2024-12-05T11:56:40,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:56:40,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/9c52c54b5eb1464196ddbcf5200ab053 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/9c52c54b5eb1464196ddbcf5200ab053 2024-12-05T11:56:40,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/638a7f1f8ab640118ad20abbdce366e5 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/638a7f1f8ab640118ad20abbdce366e5 2024-12-05T11:56:40,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/42bef86dda1e4347ad59a7907143d74a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/42bef86dda1e4347ad59a7907143d74a 2024-12-05T11:56:40,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/2b2c56fc9d3a4c70bf235dca13671c5b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/2b2c56fc9d3a4c70bf235dca13671c5b 2024-12-05T11:56:40,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/1b1d932fb18c4dd8860bb3b91386c466 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/1b1d932fb18c4dd8860bb3b91386c466 2024-12-05T11:56:40,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/540fdfbcbecd432698cc96cb00866aa6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/540fdfbcbecd432698cc96cb00866aa6 2024-12-05T11:56:40,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8118d40a82694c178f222bcc12d5d8c2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8118d40a82694c178f222bcc12d5d8c2 2024-12-05T11:56:40,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5924132f784f416fa59279e9114cfd70 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5924132f784f416fa59279e9114cfd70 2024-12-05T11:56:40,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/4d60b24621cc458393523d972fa1e6af to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/4d60b24621cc458393523d972fa1e6af 2024-12-05T11:56:40,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a2a3fdc49068424da43992a2b4b6475a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a2a3fdc49068424da43992a2b4b6475a 2024-12-05T11:56:40,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/6af9b1f02af5400fa698a357492a9689 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/6af9b1f02af5400fa698a357492a9689 2024-12-05T11:56:40,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/df856310edc445e48e7db2be7d187d78 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/df856310edc445e48e7db2be7d187d78 2024-12-05T11:56:40,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/933a33e8f2ce411d904d5aa6f0ee4775 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/933a33e8f2ce411d904d5aa6f0ee4775 2024-12-05T11:56:40,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/4e206cb8c5124c9c867304b08303329d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/4e206cb8c5124c9c867304b08303329d 2024-12-05T11:56:40,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a1ef1e6b73ab4d1084c1da523f39e3f3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a1ef1e6b73ab4d1084c1da523f39e3f3 2024-12-05T11:56:40,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/f460d083a3e34574876a74f9ab95c65d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/f460d083a3e34574876a74f9ab95c65d 2024-12-05T11:56:40,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/1da123d70b154bb8bb1744657e690169 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/1da123d70b154bb8bb1744657e690169 2024-12-05T11:56:40,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/b2764102530a4a07893aadd04740e756 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/b2764102530a4a07893aadd04740e756 2024-12-05T11:56:40,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8136405a1c744c6c8161c4f517d15a84 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8136405a1c744c6c8161c4f517d15a84 2024-12-05T11:56:40,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/abfe05618ef34326b44d5ca4c8de1b23 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/abfe05618ef34326b44d5ca4c8de1b23 2024-12-05T11:56:40,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/35311c18e4f5444aa6f0223132717d87 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/35311c18e4f5444aa6f0223132717d87 2024-12-05T11:56:40,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/88e34637c22e49c08ccb6220b11487bc to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/88e34637c22e49c08ccb6220b11487bc 2024-12-05T11:56:40,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/7bfd42af3c4340a7b09a580b1ca2242c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/7bfd42af3c4340a7b09a580b1ca2242c 2024-12-05T11:56:40,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/e7fd624ab7e2485cb5533d54dfde7221 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/e7fd624ab7e2485cb5533d54dfde7221 2024-12-05T11:56:40,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/f77cb20a22a54818a3af2c7b62213a7a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/f77cb20a22a54818a3af2c7b62213a7a 2024-12-05T11:56:40,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/c03e4b8a49f34b7b99a7ba6b958dd7ef to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/c03e4b8a49f34b7b99a7ba6b958dd7ef 2024-12-05T11:56:40,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5dc3b9f83e974058a742ee6f7a8059ad to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5dc3b9f83e974058a742ee6f7a8059ad 2024-12-05T11:56:40,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a3e90092342c483dbd6be890330594c2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/a3e90092342c483dbd6be890330594c2 2024-12-05T11:56:40,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/09baad3f38844276bcdf425bf9d9c92d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/09baad3f38844276bcdf425bf9d9c92d 2024-12-05T11:56:40,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5a488be729c34979b692a947a04b73f2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/5a488be729c34979b692a947a04b73f2 2024-12-05T11:56:40,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/d67b36150cf04237b5246b2ec67922ce to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/d67b36150cf04237b5246b2ec67922ce 2024-12-05T11:56:40,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/0341a01852204e78b01f18a47f956e40 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/0341a01852204e78b01f18a47f956e40 2024-12-05T11:56:40,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/06dc47f98c9144c6be1974d9c1e0339b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/06dc47f98c9144c6be1974d9c1e0339b 2024-12-05T11:56:40,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/7c6d4588d4114840bf7fb44a86ddaebb to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/7c6d4588d4114840bf7fb44a86ddaebb 2024-12-05T11:56:40,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/06fe62d13afa4e9ea49003a7c404913b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/4c7de984108443c2a259681bb29ed837, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/6853adb7d3a54aa2b1096e8ceac0e5f3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/374d7ce1910d492a8193e42e31bf9bb6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/94c9d54b55be4537b07e26ab4e98de0e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f08f466039f441238b0b24e56b5b513b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/fb57f307a5c44cf09007e9963fcb613c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/63791b4e5d274c17bc8ef04b50490ea6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/be69616aa13a487daecd3697973abc1d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/70c8def2bd2c4c1ebe889a19776b76bb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/a2482dec0205476da9a98aff8a3d7d6a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/0f009afc958e4981b191bf980e6bbc93, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/3cb4ba5018e14ded95f2add9317da829, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/6414cbcd310e4985ae395208a6c399be, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ad37f2e7d8f64d8db91319e0969c9dca, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/dc96fbde35d54e2a9e26f213a2a9134f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/29b0d5ab8f1c41978b9295fc0adcc6d1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ba675a18a1164530ae16ac67ad53b487, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/2511ba5ce9454eebad9d33c839fa81e0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f2459c39e0f745a4af753829a4ff237d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/0b664f0514d14be6b0a0ba75171ab0ba, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/cd2129f24dd64b04968b2bfc7eb83109, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ed6c4056a1894f359b5b22acd79702ad, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d8a197c509c54a3682a04181bf2a6430, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/5321df1c2e7140eab0dda68aff38b35c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/5450a04483da4d3aa696a0b03cc21cfe, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/53d47acdbcfa439aaf6dba6e8d134b0e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/c5780e5d71b74bce843e2b411c1fbf4f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/c2ddf0e333004da98f8ab8665721b9b9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d0deb9e499124d3bbe8550890b916ccf, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/37e24a56c29b42a18751062a6f3b9dff, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/21bd7cfb95464840bc18f465d3e5339a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/43bd7b4d4c8945b9ad4739da380490fa, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/52b6a6554ea9409ebc252a84de7140d9] to archive 2024-12-05T11:56:40,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:56:40,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/06fe62d13afa4e9ea49003a7c404913b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/06fe62d13afa4e9ea49003a7c404913b 2024-12-05T11:56:40,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/4c7de984108443c2a259681bb29ed837 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/4c7de984108443c2a259681bb29ed837 2024-12-05T11:56:40,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/6853adb7d3a54aa2b1096e8ceac0e5f3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/6853adb7d3a54aa2b1096e8ceac0e5f3 2024-12-05T11:56:40,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/374d7ce1910d492a8193e42e31bf9bb6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/374d7ce1910d492a8193e42e31bf9bb6 2024-12-05T11:56:40,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/94c9d54b55be4537b07e26ab4e98de0e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/94c9d54b55be4537b07e26ab4e98de0e 2024-12-05T11:56:40,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f08f466039f441238b0b24e56b5b513b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f08f466039f441238b0b24e56b5b513b 2024-12-05T11:56:40,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/fb57f307a5c44cf09007e9963fcb613c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/fb57f307a5c44cf09007e9963fcb613c 2024-12-05T11:56:40,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/63791b4e5d274c17bc8ef04b50490ea6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/63791b4e5d274c17bc8ef04b50490ea6 2024-12-05T11:56:40,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/be69616aa13a487daecd3697973abc1d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/be69616aa13a487daecd3697973abc1d 2024-12-05T11:56:40,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/70c8def2bd2c4c1ebe889a19776b76bb to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/70c8def2bd2c4c1ebe889a19776b76bb 2024-12-05T11:56:40,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/a2482dec0205476da9a98aff8a3d7d6a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/a2482dec0205476da9a98aff8a3d7d6a 2024-12-05T11:56:40,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/0f009afc958e4981b191bf980e6bbc93 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/0f009afc958e4981b191bf980e6bbc93 2024-12-05T11:56:40,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/3cb4ba5018e14ded95f2add9317da829 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/3cb4ba5018e14ded95f2add9317da829 2024-12-05T11:56:40,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/6414cbcd310e4985ae395208a6c399be to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/6414cbcd310e4985ae395208a6c399be 2024-12-05T11:56:40,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ad37f2e7d8f64d8db91319e0969c9dca to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ad37f2e7d8f64d8db91319e0969c9dca 2024-12-05T11:56:40,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/dc96fbde35d54e2a9e26f213a2a9134f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/dc96fbde35d54e2a9e26f213a2a9134f 2024-12-05T11:56:40,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/29b0d5ab8f1c41978b9295fc0adcc6d1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/29b0d5ab8f1c41978b9295fc0adcc6d1 2024-12-05T11:56:40,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ba675a18a1164530ae16ac67ad53b487 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ba675a18a1164530ae16ac67ad53b487 2024-12-05T11:56:40,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/2511ba5ce9454eebad9d33c839fa81e0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/2511ba5ce9454eebad9d33c839fa81e0 2024-12-05T11:56:40,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f2459c39e0f745a4af753829a4ff237d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f2459c39e0f745a4af753829a4ff237d 2024-12-05T11:56:40,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/0b664f0514d14be6b0a0ba75171ab0ba to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/0b664f0514d14be6b0a0ba75171ab0ba 2024-12-05T11:56:40,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/cd2129f24dd64b04968b2bfc7eb83109 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/cd2129f24dd64b04968b2bfc7eb83109 2024-12-05T11:56:40,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ed6c4056a1894f359b5b22acd79702ad to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/ed6c4056a1894f359b5b22acd79702ad 2024-12-05T11:56:40,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d8a197c509c54a3682a04181bf2a6430 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d8a197c509c54a3682a04181bf2a6430 2024-12-05T11:56:40,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/5321df1c2e7140eab0dda68aff38b35c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/5321df1c2e7140eab0dda68aff38b35c 2024-12-05T11:56:40,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/5450a04483da4d3aa696a0b03cc21cfe to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/5450a04483da4d3aa696a0b03cc21cfe 2024-12-05T11:56:40,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/53d47acdbcfa439aaf6dba6e8d134b0e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/53d47acdbcfa439aaf6dba6e8d134b0e 2024-12-05T11:56:40,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/c5780e5d71b74bce843e2b411c1fbf4f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/c5780e5d71b74bce843e2b411c1fbf4f 2024-12-05T11:56:40,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/c2ddf0e333004da98f8ab8665721b9b9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/c2ddf0e333004da98f8ab8665721b9b9 2024-12-05T11:56:40,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d0deb9e499124d3bbe8550890b916ccf to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d0deb9e499124d3bbe8550890b916ccf 2024-12-05T11:56:40,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/37e24a56c29b42a18751062a6f3b9dff to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/37e24a56c29b42a18751062a6f3b9dff 2024-12-05T11:56:40,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/21bd7cfb95464840bc18f465d3e5339a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/21bd7cfb95464840bc18f465d3e5339a 2024-12-05T11:56:40,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/43bd7b4d4c8945b9ad4739da380490fa to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/43bd7b4d4c8945b9ad4739da380490fa 2024-12-05T11:56:40,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/52b6a6554ea9409ebc252a84de7140d9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/52b6a6554ea9409ebc252a84de7140d9 2024-12-05T11:56:40,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/996a80e8380f40f8ac97588d711d948e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/38aabd8152b24cf3b047d4c9782fcad5, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/2550e03d4e5f4298880a8f4184c2c84b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/c53d8ee1b7884d0ca6fb0b205bc621db, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e6769e4d65984cf0b94c0cbeb3fd426e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/af5d094374d64f85a6669bce8c8ce29f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/4f24d51eb4e8419db5d420131de99a61, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/6991b238a930405bad403f2541b08749, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a6e57132ef0f40eda068ec6b2c2ec001, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e08a989c584d4333b06a69ff39a3bc89, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/882ff189accc4258bc57ad2a9038c900, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/f2dd0f5123e440248e1646a8ca15b26d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a7cf3e7d100144448ce40903561a663a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/d5867cc8804c43d6b21d3e61111e426e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/db33ceaee58f4a0c96c57b3f41d81504, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/895495768fe54a7e8351040add0a697a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/08375032446445449e7e628a7d43052e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/bffcc26541aa44c4a1068ac0f87ad2d6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a940eb1f544540f6a6abdcbaff87707b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e86272253f434f7d8b64ec90457f77cf, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/64557fd9206c4a558c39c6febdcd80bd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/0e7b8e51379f44eb924f286a956c3b55, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/25df00a5a50b4feab506511c31533afd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/49e65b3660da48979918c6e992f5a0b1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/63e7e2dd5b3947d9a31b8450a171bb7d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/0898a695f47d4dbf9321620ffdab46b1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/796b76a32dc545aca7befcf3fcb0c827, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/9bcb0c3bfc724d1c8a7db6188695d393, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/5918f0921c0c4e7091199a5e03c151c0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/ee98d8374c8d4fe5af2e097a174c5750, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/ab251302cd1f4f82ae3cad9c55540ef8, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/d09a1ec349014e0a9494ff4754b65af6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/3f3297c10157460d9ae0e9c4c388c3fe, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/6f7511dc08364a7094b051afb47aff19] to archive 2024-12-05T11:56:40,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:56:40,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/996a80e8380f40f8ac97588d711d948e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/996a80e8380f40f8ac97588d711d948e 2024-12-05T11:56:40,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/38aabd8152b24cf3b047d4c9782fcad5 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/38aabd8152b24cf3b047d4c9782fcad5 2024-12-05T11:56:40,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/2550e03d4e5f4298880a8f4184c2c84b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/2550e03d4e5f4298880a8f4184c2c84b 2024-12-05T11:56:40,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/c53d8ee1b7884d0ca6fb0b205bc621db to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/c53d8ee1b7884d0ca6fb0b205bc621db 2024-12-05T11:56:40,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e6769e4d65984cf0b94c0cbeb3fd426e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e6769e4d65984cf0b94c0cbeb3fd426e 2024-12-05T11:56:40,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/af5d094374d64f85a6669bce8c8ce29f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/af5d094374d64f85a6669bce8c8ce29f 2024-12-05T11:56:40,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/4f24d51eb4e8419db5d420131de99a61 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/4f24d51eb4e8419db5d420131de99a61 2024-12-05T11:56:40,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/6991b238a930405bad403f2541b08749 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/6991b238a930405bad403f2541b08749 2024-12-05T11:56:40,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a6e57132ef0f40eda068ec6b2c2ec001 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a6e57132ef0f40eda068ec6b2c2ec001 2024-12-05T11:56:40,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e08a989c584d4333b06a69ff39a3bc89 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e08a989c584d4333b06a69ff39a3bc89 2024-12-05T11:56:40,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/882ff189accc4258bc57ad2a9038c900 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/882ff189accc4258bc57ad2a9038c900 2024-12-05T11:56:40,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/f2dd0f5123e440248e1646a8ca15b26d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/f2dd0f5123e440248e1646a8ca15b26d 2024-12-05T11:56:40,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a7cf3e7d100144448ce40903561a663a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a7cf3e7d100144448ce40903561a663a 2024-12-05T11:56:40,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/d5867cc8804c43d6b21d3e61111e426e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/d5867cc8804c43d6b21d3e61111e426e 2024-12-05T11:56:40,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/db33ceaee58f4a0c96c57b3f41d81504 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/db33ceaee58f4a0c96c57b3f41d81504 2024-12-05T11:56:40,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/895495768fe54a7e8351040add0a697a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/895495768fe54a7e8351040add0a697a 2024-12-05T11:56:40,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/08375032446445449e7e628a7d43052e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/08375032446445449e7e628a7d43052e 2024-12-05T11:56:40,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/bffcc26541aa44c4a1068ac0f87ad2d6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/bffcc26541aa44c4a1068ac0f87ad2d6 2024-12-05T11:56:40,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a940eb1f544540f6a6abdcbaff87707b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/a940eb1f544540f6a6abdcbaff87707b 2024-12-05T11:56:40,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e86272253f434f7d8b64ec90457f77cf to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/e86272253f434f7d8b64ec90457f77cf 2024-12-05T11:56:40,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/64557fd9206c4a558c39c6febdcd80bd to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/64557fd9206c4a558c39c6febdcd80bd 2024-12-05T11:56:40,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/0e7b8e51379f44eb924f286a956c3b55 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/0e7b8e51379f44eb924f286a956c3b55 2024-12-05T11:56:40,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/25df00a5a50b4feab506511c31533afd to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/25df00a5a50b4feab506511c31533afd 2024-12-05T11:56:40,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/49e65b3660da48979918c6e992f5a0b1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/49e65b3660da48979918c6e992f5a0b1 2024-12-05T11:56:40,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/63e7e2dd5b3947d9a31b8450a171bb7d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/63e7e2dd5b3947d9a31b8450a171bb7d 2024-12-05T11:56:40,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/0898a695f47d4dbf9321620ffdab46b1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/0898a695f47d4dbf9321620ffdab46b1 2024-12-05T11:56:40,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/796b76a32dc545aca7befcf3fcb0c827 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/796b76a32dc545aca7befcf3fcb0c827 2024-12-05T11:56:40,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/9bcb0c3bfc724d1c8a7db6188695d393 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/9bcb0c3bfc724d1c8a7db6188695d393 2024-12-05T11:56:40,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/5918f0921c0c4e7091199a5e03c151c0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/5918f0921c0c4e7091199a5e03c151c0 2024-12-05T11:56:40,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/ee98d8374c8d4fe5af2e097a174c5750 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/ee98d8374c8d4fe5af2e097a174c5750 2024-12-05T11:56:40,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/ab251302cd1f4f82ae3cad9c55540ef8 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/ab251302cd1f4f82ae3cad9c55540ef8 2024-12-05T11:56:40,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/d09a1ec349014e0a9494ff4754b65af6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/d09a1ec349014e0a9494ff4754b65af6 2024-12-05T11:56:40,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/3f3297c10157460d9ae0e9c4c388c3fe to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/3f3297c10157460d9ae0e9c4c388c3fe 2024-12-05T11:56:40,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/6f7511dc08364a7094b051afb47aff19 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/6f7511dc08364a7094b051afb47aff19 2024-12-05T11:56:40,319 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/recovered.edits/504.seqid, newMaxSeqId=504, maxSeqId=1 2024-12-05T11:56:40,320 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. 2024-12-05T11:56:40,320 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 41e385893d26f932b2fcba15c0b7f150: Waiting for close lock at 1733399798919Running coprocessor pre-close hooks at 1733399798919Disabling compacts and flushes for region at 1733399798919Disabling writes for close at 1733399798919Obtaining lock to block concurrent updates at 1733399798920 (+1 ms)Preparing flush snapshotting stores in 41e385893d26f932b2fcba15c0b7f150 at 1733399798920Finished memstore snapshotting TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733399798920Flushing stores of TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150. at 1733399798922 (+2 ms)Flushing 41e385893d26f932b2fcba15c0b7f150/A: creating writer at 1733399798922Flushing 41e385893d26f932b2fcba15c0b7f150/A: appending metadata at 1733399798926 (+4 ms)Flushing 41e385893d26f932b2fcba15c0b7f150/A: closing flushed file at 1733399798926Flushing 41e385893d26f932b2fcba15c0b7f150/B: creating writer at 1733399799336 (+410 ms)Flushing 41e385893d26f932b2fcba15c0b7f150/B: appending metadata at 1733399799339 (+3 ms)Flushing 41e385893d26f932b2fcba15c0b7f150/B: closing flushed file at 1733399799339Flushing 41e385893d26f932b2fcba15c0b7f150/C: creating writer at 1733399799750 (+411 ms)Flushing 41e385893d26f932b2fcba15c0b7f150/C: appending metadata at 1733399799752 (+2 ms)Flushing 41e385893d26f932b2fcba15c0b7f150/C: closing flushed file at 1733399799752Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d3326fb: reopening flushed file at 1733399800164 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3363d8cd: reopening flushed file at 1733399800169 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48fc6911: reopening flushed file at 1733399800174 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 41e385893d26f932b2fcba15c0b7f150 in 1261ms, sequenceid=501, compaction requested=true at 1733399800180 (+6 ms)Writing region close event to WAL at 1733399800315 (+135 ms)Running coprocessor post-close hooks at 1733399800320 (+5 ms)Closed at 1733399800320 2024-12-05T11:56:40,322 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:40,323 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=41e385893d26f932b2fcba15c0b7f150, regionState=CLOSED 2024-12-05T11:56:40,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:56:40,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-05T11:56:40,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 41e385893d26f932b2fcba15c0b7f150, server=80666a11a09f,40121,1733399769572 in 1.5690 sec 2024-12-05T11:56:40,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-05T11:56:40,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41e385893d26f932b2fcba15c0b7f150, UNASSIGN in 1.5750 sec 2024-12-05T11:56:40,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-05T11:56:40,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5830 sec 2024-12-05T11:56:40,333 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399800332"}]},"ts":"1733399800332"} 2024-12-05T11:56:40,334 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T11:56:40,334 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T11:56:40,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5970 sec 2024-12-05T11:56:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T11:56:40,888 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:40,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T11:56:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:56:40,896 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:56:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-05T11:56:40,898 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:56:40,902 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:40,907 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/recovered.edits] 2024-12-05T11:56:40,911 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8c13571057dd457196bdde5ff98440f3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/8c13571057dd457196bdde5ff98440f3 2024-12-05T11:56:40,913 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/ceb7fef73b204c37a0e139f6a87a4177 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/ceb7fef73b204c37a0e139f6a87a4177 2024-12-05T11:56:40,915 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/ea68617f6b30474baf85a5f2a7e54fdd to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/A/ea68617f6b30474baf85a5f2a7e54fdd 2024-12-05T11:56:40,919 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/227a736096484a8989cffad98ab87998 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/227a736096484a8989cffad98ab87998 2024-12-05T11:56:40,920 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d69de72416c841d89373f6d251281f9e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/d69de72416c841d89373f6d251281f9e 2024-12-05T11:56:40,921 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f3679883e62b435c884121bc6ea233d2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/B/f3679883e62b435c884121bc6ea233d2 2024-12-05T11:56:40,923 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/026aabca66ac4d5882f313f85449290d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/026aabca66ac4d5882f313f85449290d 2024-12-05T11:56:40,924 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/60f6f1e5960e4e12923a88e37f2418a8 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/60f6f1e5960e4e12923a88e37f2418a8 2024-12-05T11:56:40,925 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/f93caab3ec494b1e85669e1303f23c28 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/C/f93caab3ec494b1e85669e1303f23c28 2024-12-05T11:56:40,928 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/recovered.edits/504.seqid to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150/recovered.edits/504.seqid 2024-12-05T11:56:40,929 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/41e385893d26f932b2fcba15c0b7f150 2024-12-05T11:56:40,929 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T11:56:40,933 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:56:40,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-05T11:56:40,941 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T11:56:40,965 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T11:56:40,966 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:56:40,966 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T11:56:40,966 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733399800966"}]},"ts":"9223372036854775807"} 2024-12-05T11:56:40,971 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T11:56:40,971 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 41e385893d26f932b2fcba15c0b7f150, NAME => 'TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T11:56:40,971 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T11:56:40,971 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733399800971"}]},"ts":"9223372036854775807"} 2024-12-05T11:56:40,975 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T11:56:40,976 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:56:40,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-12-05T11:56:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-05T11:56:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T11:56:41,009 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:41,027 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/80666a11a09f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1451530749_22 at /127.0.0.1:52172 [Waiting for operation #355] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1849376751_22 at /127.0.0.1:52314 [Waiting for operation #334] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;80666a11a09f:40121-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=288 (was 223) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6909 (was 7423) 2024-12-05T11:56:41,036 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=288, ProcessCount=11, AvailableMemoryMB=6909 2024-12-05T11:56:41,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:56:41,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:56:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T11:56:41,040 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T11:56:41,040 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:41,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-12-05T11:56:41,041 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T11:56:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-05T11:56:41,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741948_1124 (size=959) 2024-12-05T11:56:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-05T11:56:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-05T11:56:41,468 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c 2024-12-05T11:56:41,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741949_1125 (size=53) 2024-12-05T11:56:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-05T11:56:41,879 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:56:41,879 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cde5c157572c2beb0bd959b68c61d4ee, disabling compactions & flushes 2024-12-05T11:56:41,879 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:41,879 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:41,879 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. after waiting 0 ms 2024-12-05T11:56:41,879 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:41,879 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:41,879 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cde5c157572c2beb0bd959b68c61d4ee: Waiting for close lock at 1733399801879Disabling compacts and flushes for region at 1733399801879Disabling writes for close at 1733399801879Writing region close event to WAL at 1733399801879Closed at 1733399801879 2024-12-05T11:56:41,880 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T11:56:41,880 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733399801880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733399801880"}]},"ts":"1733399801880"} 2024-12-05T11:56:41,882 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T11:56:41,883 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T11:56:41,883 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399801883"}]},"ts":"1733399801883"} 2024-12-05T11:56:41,884 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T11:56:41,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, ASSIGN}] 2024-12-05T11:56:41,886 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, ASSIGN 2024-12-05T11:56:41,887 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, ASSIGN; state=OFFLINE, location=80666a11a09f,40121,1733399769572; forceNewPlan=false, retain=false 2024-12-05T11:56:42,037 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=cde5c157572c2beb0bd959b68c61d4ee, regionState=OPENING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:56:42,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, ASSIGN because future has completed 2024-12-05T11:56:42,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:56:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-05T11:56:42,196 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:42,196 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:56:42,197 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,197 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:56:42,197 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,197 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,198 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,200 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cde5c157572c2beb0bd959b68c61d4ee columnFamilyName A 2024-12-05T11:56:42,201 DEBUG [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:42,201 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] regionserver.HStore(327): Store=cde5c157572c2beb0bd959b68c61d4ee/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:56:42,202 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,203 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cde5c157572c2beb0bd959b68c61d4ee columnFamilyName B 2024-12-05T11:56:42,203 DEBUG [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:42,204 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] regionserver.HStore(327): Store=cde5c157572c2beb0bd959b68c61d4ee/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:56:42,204 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,205 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cde5c157572c2beb0bd959b68c61d4ee columnFamilyName C 2024-12-05T11:56:42,205 DEBUG [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:42,205 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] regionserver.HStore(327): Store=cde5c157572c2beb0bd959b68c61d4ee/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:56:42,205 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,206 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,206 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,207 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,207 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,208 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:56:42,209 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,211 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:56:42,212 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened cde5c157572c2beb0bd959b68c61d4ee; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73512126, jitterRate=0.09541603922843933}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:56:42,212 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:42,213 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for cde5c157572c2beb0bd959b68c61d4ee: Running coprocessor pre-open hook at 1733399802197Writing region info on filesystem at 1733399802197Initializing all the Stores at 1733399802198 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399802198Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399802198Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399802198Cleaning up temporary data from old regions at 1733399802207 (+9 ms)Running coprocessor post-open hooks at 1733399802212 (+5 ms)Region opened successfully at 1733399802213 (+1 ms) 2024-12-05T11:56:42,214 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., pid=34, masterSystemTime=1733399802192 2024-12-05T11:56:42,217 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:42,217 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:42,218 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=cde5c157572c2beb0bd959b68c61d4ee, regionState=OPEN, openSeqNum=2, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:56:42,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:56:42,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-05T11:56:42,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 in 196 msec 2024-12-05T11:56:42,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-05T11:56:42,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, ASSIGN in 353 msec 2024-12-05T11:56:42,240 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T11:56:42,241 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399802240"}]},"ts":"1733399802240"} 2024-12-05T11:56:42,242 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T11:56:42,243 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T11:56:42,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2050 sec 2024-12-05T11:56:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-05T11:56:43,188 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:43,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:43,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:43,191 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:43,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:43,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:43,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295e4063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:43,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:43,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:43,193 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:43,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd695c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,194 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:43,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:43,197 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:56:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 80666a11a09f,33741,1733399768894 2024-12-05T11:56:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c72d130 2024-12-05T11:56:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:56:43,200 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:56:43,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:56:43,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:56:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T11:56:43,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741950_1126 (size=995) 2024-12-05T11:56:43,221 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-05T11:56:43,221 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-05T11:56:43,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:56:43,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, REOPEN/MOVE}] 2024-12-05T11:56:43,231 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, REOPEN/MOVE 2024-12-05T11:56:43,231 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cde5c157572c2beb0bd959b68c61d4ee, regionState=CLOSING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:56:43,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, REOPEN/MOVE because future has completed 2024-12-05T11:56:43,233 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:56:43,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:56:43,386 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,386 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:56:43,386 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing cde5c157572c2beb0bd959b68c61d4ee, disabling compactions & flushes 2024-12-05T11:56:43,386 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:43,386 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:43,386 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. after waiting 0 ms 2024-12-05T11:56:43,386 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:43,390 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T11:56:43,390 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:43,390 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for cde5c157572c2beb0bd959b68c61d4ee: Waiting for close lock at 1733399803386Running coprocessor pre-close hooks at 1733399803386Disabling compacts and flushes for region at 1733399803386Disabling writes for close at 1733399803386Writing region close event to WAL at 1733399803386Running coprocessor post-close hooks at 1733399803390 (+4 ms)Closed at 1733399803390 2024-12-05T11:56:43,390 WARN [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: cde5c157572c2beb0bd959b68c61d4ee to self. 2024-12-05T11:56:43,392 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,393 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cde5c157572c2beb0bd959b68c61d4ee, regionState=CLOSED 2024-12-05T11:56:43,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:56:43,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-05T11:56:43,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 in 163 msec 2024-12-05T11:56:43,398 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, REOPEN/MOVE; state=CLOSED, location=80666a11a09f,40121,1733399769572; forceNewPlan=false, retain=true 2024-12-05T11:56:43,549 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cde5c157572c2beb0bd959b68c61d4ee, regionState=OPENING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:56:43,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, REOPEN/MOVE because future has completed 2024-12-05T11:56:43,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:56:43,733 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:43,733 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:56:43,733 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,734 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:56:43,734 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,734 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,737 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,742 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cde5c157572c2beb0bd959b68c61d4ee columnFamilyName A 2024-12-05T11:56:43,744 DEBUG [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:43,745 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] regionserver.HStore(327): Store=cde5c157572c2beb0bd959b68c61d4ee/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:56:43,745 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,746 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cde5c157572c2beb0bd959b68c61d4ee columnFamilyName B 2024-12-05T11:56:43,746 DEBUG [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:43,747 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] regionserver.HStore(327): Store=cde5c157572c2beb0bd959b68c61d4ee/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:56:43,747 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,747 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cde5c157572c2beb0bd959b68c61d4ee columnFamilyName C 2024-12-05T11:56:43,748 DEBUG [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:43,748 INFO [StoreOpener-cde5c157572c2beb0bd959b68c61d4ee-1 {}] regionserver.HStore(327): Store=cde5c157572c2beb0bd959b68c61d4ee/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:56:43,748 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,749 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,750 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,751 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,751 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,751 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:56:43,752 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,754 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened cde5c157572c2beb0bd959b68c61d4ee; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64399119, jitterRate=-0.04037834703922272}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:56:43,754 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,754 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for cde5c157572c2beb0bd959b68c61d4ee: Running coprocessor pre-open hook at 1733399803734Writing region info on filesystem at 1733399803734Initializing all the Stores at 1733399803734Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399803734Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399803735 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399803735Cleaning up temporary data from old regions at 1733399803751 (+16 ms)Running coprocessor post-open hooks at 1733399803754 (+3 ms)Region opened successfully at 1733399803754 2024-12-05T11:56:43,755 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., pid=39, masterSystemTime=1733399803730 2024-12-05T11:56:43,757 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:43,757 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:43,757 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cde5c157572c2beb0bd959b68c61d4ee, regionState=OPEN, openSeqNum=5, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:56:43,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:56:43,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-12-05T11:56:43,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 in 207 msec 2024-12-05T11:56:43,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-05T11:56:43,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, REOPEN/MOVE in 532 msec 2024-12-05T11:56:43,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-05T11:56:43,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 541 msec 2024-12-05T11:56:43,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 562 msec 2024-12-05T11:56:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-05T11:56:43,772 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:43,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:43,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:43,774 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d5b5ec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:43,776 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:43,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554d2b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:43,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:43,779 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30213712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:43,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:43,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:43,781 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c4df5fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:43,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:43,783 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d09233e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:43,785 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9306ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,786 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:43,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:43,787 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4efec68d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:43,788 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fbb28b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:43,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:43,790 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb0b254, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:43,791 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517b9dcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:43,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:43,794 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1605a4f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:43,795 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b956a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:43,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:43,797 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f42cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:43,798 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:43,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@701031bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:43,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:43,800 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e90d2cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:43,802 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1701504c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:56:43,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:56:43,804 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:56:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:56:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:56:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70cbebb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:56:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:56:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:56:43,806 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:56:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a87beba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:56:43,807 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:43,807 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:43,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:43,807 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:43,807 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-05T11:56:43,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:43,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:43,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:43,810 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:43,810 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:43,811 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:56:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:43,812 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:43,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:43,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:43,814 DEBUG [Thread-640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:43,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:43,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:56:43,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:56:43,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:56:43,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:56:43,820 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:43,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:56:43,820 DEBUG [Thread-632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:56:43,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:56:43,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:43,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:56:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:56:43,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:43,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:43,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:56:43,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:56:43,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:56:43,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:56:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:43,832 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:43,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T11:56:43,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057ce52aa74ca14ff5acbd27a12fb66caf_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399803831/Put/seqid=0 2024-12-05T11:56:43,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741951_1127 (size=12154) 2024-12-05T11:56:43,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399863880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:43,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399863882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:43,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399863884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:43,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399863907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:43,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:43,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:43,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:43,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399863907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:43,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:43,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:43,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:56:43,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:56:43,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:43,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:43,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:43,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:43,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399864008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399864008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399864018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399864018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399864023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:56:44,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:44,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:44,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:44,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:56:44,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399864218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399864218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399864228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399864228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399864228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:56:44,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:44,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:44,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:44,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,304 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:44,310 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057ce52aa74ca14ff5acbd27a12fb66caf_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057ce52aa74ca14ff5acbd27a12fb66caf_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:44,313 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/20afad4dc7734efeb91fd50257838f2e, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:44,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/20afad4dc7734efeb91fd50257838f2e is 175, key is test_row_0/A:col10/1733399803831/Put/seqid=0 2024-12-05T11:56:44,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741952_1128 (size=30955) 2024-12-05T11:56:44,344 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/20afad4dc7734efeb91fd50257838f2e 2024-12-05T11:56:44,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/0fd5f3ec9d004798959b77b671833e21 is 50, key is test_row_0/B:col10/1733399803831/Put/seqid=0 2024-12-05T11:56:44,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741953_1129 (size=12001) 2024-12-05T11:56:44,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/0fd5f3ec9d004798959b77b671833e21 2024-12-05T11:56:44,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:56:44,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:44,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:44,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:44,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:56:44,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/5ed68ff89c2e4c20811aa98f8779e10a is 50, key is test_row_0/C:col10/1733399803831/Put/seqid=0 2024-12-05T11:56:44,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741954_1130 (size=12001) 2024-12-05T11:56:44,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/5ed68ff89c2e4c20811aa98f8779e10a 2024-12-05T11:56:44,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/20afad4dc7734efeb91fd50257838f2e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/20afad4dc7734efeb91fd50257838f2e 2024-12-05T11:56:44,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/20afad4dc7734efeb91fd50257838f2e, entries=150, sequenceid=16, filesize=30.2 K 2024-12-05T11:56:44,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/0fd5f3ec9d004798959b77b671833e21 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0fd5f3ec9d004798959b77b671833e21 2024-12-05T11:56:44,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0fd5f3ec9d004798959b77b671833e21, entries=150, sequenceid=16, filesize=11.7 K 2024-12-05T11:56:44,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/5ed68ff89c2e4c20811aa98f8779e10a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/5ed68ff89c2e4c20811aa98f8779e10a 2024-12-05T11:56:44,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/5ed68ff89c2e4c20811aa98f8779e10a, entries=150, sequenceid=16, filesize=11.7 K 2024-12-05T11:56:44,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for cde5c157572c2beb0bd959b68c61d4ee in 681ms, sequenceid=16, compaction requested=false 2024-12-05T11:56:44,513 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T11:56:44,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:44,531 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:44,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-05T11:56:44,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205374f251f387b468fbd792744c8f27d2a_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399804528/Put/seqid=0 2024-12-05T11:56:44,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741955_1131 (size=12154) 2024-12-05T11:56:44,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:56:44,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:44,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:44,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:44,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399864579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399864582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399864583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399864583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399864593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399864698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399864698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399864698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399864698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399864708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:56:44,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:44,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:44,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:44,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:56:44,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:44,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:44,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:44,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:44,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399864908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399864908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399864908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399864908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399864917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:44,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:56:44,968 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:44,975 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205374f251f387b468fbd792744c8f27d2a_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205374f251f387b468fbd792744c8f27d2a_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:44,977 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/1078b9cf6e4e4239be9c92b4fe1d4b76, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:44,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/1078b9cf6e4e4239be9c92b4fe1d4b76 is 175, key is test_row_0/A:col10/1733399804528/Put/seqid=0 2024-12-05T11:56:45,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741956_1132 (size=30955) 2024-12-05T11:56:45,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:56:45,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:45,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:45,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:45,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:45,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:45,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:56:45,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:45,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:45,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:45,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:45,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:45,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399865218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399865218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399865218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399865218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399865228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:56:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:45,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:45,406 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=43, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/1078b9cf6e4e4239be9c92b4fe1d4b76 2024-12-05T11:56:45,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/41b1a6858fd241298f2ee186037ed9dd is 50, key is test_row_0/B:col10/1733399804528/Put/seqid=0 2024-12-05T11:56:45,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741957_1133 (size=12001) 2024-12-05T11:56:45,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/41b1a6858fd241298f2ee186037ed9dd 2024-12-05T11:56:45,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/c463e01d23484ec098499b01c66d718b is 50, key is test_row_0/C:col10/1733399804528/Put/seqid=0 2024-12-05T11:56:45,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741958_1134 (size=12001) 2024-12-05T11:56:45,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/c463e01d23484ec098499b01c66d718b 2024-12-05T11:56:45,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:56:45,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:45,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:45,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:45,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:45,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:45,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/1078b9cf6e4e4239be9c92b4fe1d4b76 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/1078b9cf6e4e4239be9c92b4fe1d4b76 2024-12-05T11:56:45,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/1078b9cf6e4e4239be9c92b4fe1d4b76, entries=150, sequenceid=43, filesize=30.2 K 2024-12-05T11:56:45,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/41b1a6858fd241298f2ee186037ed9dd as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/41b1a6858fd241298f2ee186037ed9dd 2024-12-05T11:56:45,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/41b1a6858fd241298f2ee186037ed9dd, entries=150, sequenceid=43, filesize=11.7 K 2024-12-05T11:56:45,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/c463e01d23484ec098499b01c66d718b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c463e01d23484ec098499b01c66d718b 2024-12-05T11:56:45,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c463e01d23484ec098499b01c66d718b, entries=150, sequenceid=43, filesize=11.7 K 2024-12-05T11:56:45,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for cde5c157572c2beb0bd959b68c61d4ee in 1031ms, sequenceid=43, compaction requested=false 2024-12-05T11:56:45,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:45,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:56:45,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:45,661 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-05T11:56:45,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205414facfcf42d478080867bde102d17b4_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399804590/Put/seqid=0 2024-12-05T11:56:45,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741959_1135 (size=12154) 2024-12-05T11:56:45,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:45,695 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205414facfcf42d478080867bde102d17b4_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205414facfcf42d478080867bde102d17b4_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:45,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/dea7b107d5444ed0bbcb7ee15502e1fb, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:45,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/dea7b107d5444ed0bbcb7ee15502e1fb is 175, key is test_row_0/A:col10/1733399804590/Put/seqid=0 2024-12-05T11:56:45,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741960_1136 (size=30955) 2024-12-05T11:56:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:45,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:45,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399865768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399865768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399865772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399865772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399865774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399865878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399865878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399865888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399865888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399865888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:45,890 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:56:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:56:46,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399866088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399866088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399866098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399866098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399866098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,110 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/dea7b107d5444ed0bbcb7ee15502e1fb 2024-12-05T11:56:46,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/a40ac5c6444c4d3bb0bb47886029b5b7 is 50, key is test_row_0/B:col10/1733399804590/Put/seqid=0 2024-12-05T11:56:46,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741961_1137 (size=12001) 2024-12-05T11:56:46,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399866398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399866397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399866408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399866418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399866418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,542 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/a40ac5c6444c4d3bb0bb47886029b5b7 2024-12-05T11:56:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/c0f4669b1e154c2f8a20fc3244b6bc0e is 50, key is test_row_0/C:col10/1733399804590/Put/seqid=0 2024-12-05T11:56:46,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741962_1138 (size=12001) 2024-12-05T11:56:46,597 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/c0f4669b1e154c2f8a20fc3244b6bc0e 2024-12-05T11:56:46,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/dea7b107d5444ed0bbcb7ee15502e1fb as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/dea7b107d5444ed0bbcb7ee15502e1fb 2024-12-05T11:56:46,619 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/dea7b107d5444ed0bbcb7ee15502e1fb, entries=150, sequenceid=52, filesize=30.2 K 2024-12-05T11:56:46,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/a40ac5c6444c4d3bb0bb47886029b5b7 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/a40ac5c6444c4d3bb0bb47886029b5b7 2024-12-05T11:56:46,627 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/a40ac5c6444c4d3bb0bb47886029b5b7, entries=150, sequenceid=52, filesize=11.7 K 2024-12-05T11:56:46,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/c0f4669b1e154c2f8a20fc3244b6bc0e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c0f4669b1e154c2f8a20fc3244b6bc0e 2024-12-05T11:56:46,642 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c0f4669b1e154c2f8a20fc3244b6bc0e, entries=150, sequenceid=52, filesize=11.7 K 2024-12-05T11:56:46,645 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for cde5c157572c2beb0bd959b68c61d4ee in 984ms, sequenceid=52, compaction requested=true 2024-12-05T11:56:46,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:46,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:46,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-05T11:56:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-05T11:56:46,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-05T11:56:46,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8360 sec 2024-12-05T11:56:46,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.8450 sec 2024-12-05T11:56:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:46,910 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:46,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-12-05T11:56:46,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399866915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399866918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399866920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120542ad50be917e478abd304832c58694cf_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399805770/Put/seqid=0 2024-12-05T11:56:46,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399866928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399866928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:46,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:46,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:46,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741963_1139 (size=14594) 2024-12-05T11:56:46,981 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:46,987 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120542ad50be917e478abd304832c58694cf_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120542ad50be917e478abd304832c58694cf_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:46,990 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/6a7e372ec89846639f19a7b1704334a1, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:46,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/6a7e372ec89846639f19a7b1704334a1 is 175, key is test_row_0/A:col10/1733399805770/Put/seqid=0 2024-12-05T11:56:46,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741964_1140 (size=39549) 2024-12-05T11:56:47,000 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/6a7e372ec89846639f19a7b1704334a1 2024-12-05T11:56:47,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/125026c1b21b43fc86ad5e339877c220 is 50, key is test_row_0/B:col10/1733399805770/Put/seqid=0 2024-12-05T11:56:47,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741965_1141 (size=12001) 2024-12-05T11:56:47,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399867028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399867029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399867237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399867238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/125026c1b21b43fc86ad5e339877c220 2024-12-05T11:56:47,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/7de4d61bbcff4b139e7cc31ef61977ad is 50, key is test_row_0/C:col10/1733399805770/Put/seqid=0 2024-12-05T11:56:47,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741966_1142 (size=12001) 2024-12-05T11:56:47,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/7de4d61bbcff4b139e7cc31ef61977ad 2024-12-05T11:56:47,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/6a7e372ec89846639f19a7b1704334a1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6a7e372ec89846639f19a7b1704334a1 2024-12-05T11:56:47,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6a7e372ec89846639f19a7b1704334a1, entries=200, sequenceid=82, filesize=38.6 K 2024-12-05T11:56:47,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/125026c1b21b43fc86ad5e339877c220 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/125026c1b21b43fc86ad5e339877c220 2024-12-05T11:56:47,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/125026c1b21b43fc86ad5e339877c220, entries=150, sequenceid=82, filesize=11.7 K 2024-12-05T11:56:47,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/7de4d61bbcff4b139e7cc31ef61977ad as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/7de4d61bbcff4b139e7cc31ef61977ad 2024-12-05T11:56:47,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/7de4d61bbcff4b139e7cc31ef61977ad, entries=150, sequenceid=82, filesize=11.7 K 2024-12-05T11:56:47,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for cde5c157572c2beb0bd959b68c61d4ee in 575ms, sequenceid=82, compaction requested=true 2024-12-05T11:56:47,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:47,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:47,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:47,485 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:56:47,485 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:56:47,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:47,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:47,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:47,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:47,487 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:56:47,487 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/B is initiating minor compaction (all files) 2024-12-05T11:56:47,487 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/B in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:47,487 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0fd5f3ec9d004798959b77b671833e21, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/41b1a6858fd241298f2ee186037ed9dd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/a40ac5c6444c4d3bb0bb47886029b5b7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/125026c1b21b43fc86ad5e339877c220] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=46.9 K 2024-12-05T11:56:47,488 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:56:47,488 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/A is initiating minor compaction (all files) 2024-12-05T11:56:47,488 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/A in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:47,488 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/20afad4dc7734efeb91fd50257838f2e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/1078b9cf6e4e4239be9c92b4fe1d4b76, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/dea7b107d5444ed0bbcb7ee15502e1fb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6a7e372ec89846639f19a7b1704334a1] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=129.3 K 2024-12-05T11:56:47,488 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:47,488 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/20afad4dc7734efeb91fd50257838f2e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/1078b9cf6e4e4239be9c92b4fe1d4b76, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/dea7b107d5444ed0bbcb7ee15502e1fb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6a7e372ec89846639f19a7b1704334a1] 2024-12-05T11:56:47,491 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20afad4dc7734efeb91fd50257838f2e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733399803828 2024-12-05T11:56:47,491 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1078b9cf6e4e4239be9c92b4fe1d4b76, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733399803881 2024-12-05T11:56:47,491 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting dea7b107d5444ed0bbcb7ee15502e1fb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399804555 2024-12-05T11:56:47,492 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a7e372ec89846639f19a7b1704334a1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733399805764 2024-12-05T11:56:47,494 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fd5f3ec9d004798959b77b671833e21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733399803828 2024-12-05T11:56:47,495 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 41b1a6858fd241298f2ee186037ed9dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733399803881 2024-12-05T11:56:47,496 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting a40ac5c6444c4d3bb0bb47886029b5b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399804555 2024-12-05T11:56:47,497 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 125026c1b21b43fc86ad5e339877c220, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733399805764 2024-12-05T11:56:47,515 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:47,529 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:47,530 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/2d5edb4777a94743bd44c958b414ba86 is 50, key is test_row_0/B:col10/1733399805770/Put/seqid=0 2024-12-05T11:56:47,540 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205a94fb1127b044f5cb2152ed526525e6c_cde5c157572c2beb0bd959b68c61d4ee store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:47,545 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205a94fb1127b044f5cb2152ed526525e6c_cde5c157572c2beb0bd959b68c61d4ee, store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:47,545 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a94fb1127b044f5cb2152ed526525e6c_cde5c157572c2beb0bd959b68c61d4ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:47,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741967_1143 (size=12139) 2024-12-05T11:56:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:47,563 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:47,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T11:56:47,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058a50933ccc1f4cb5bc9856dba4209eea_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399807551/Put/seqid=0 2024-12-05T11:56:47,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741968_1144 (size=4469) 2024-12-05T11:56:47,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741969_1145 (size=14594) 2024-12-05T11:56:47,588 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:47,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058a50933ccc1f4cb5bc9856dba4209eea_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058a50933ccc1f4cb5bc9856dba4209eea_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:47,600 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/46b598e910f1442b874912f2ee77fa10, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:47,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/46b598e910f1442b874912f2ee77fa10 is 175, key is test_row_0/A:col10/1733399807551/Put/seqid=0 2024-12-05T11:56:47,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741970_1146 (size=39549) 2024-12-05T11:56:47,624 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/46b598e910f1442b874912f2ee77fa10 2024-12-05T11:56:47,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399867628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399867631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/e173cbf94fcf415ea04eab72806ae1e6 is 50, key is test_row_0/B:col10/1733399807551/Put/seqid=0 2024-12-05T11:56:47,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741971_1147 (size=12001) 2024-12-05T11:56:47,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399867737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399867737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399867928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,929 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in cde5c157572c2beb0bd959b68c61d4ee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399867947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399867948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399867948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399867948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:47,949 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in cde5c157572c2beb0bd959b68c61d4ee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,951 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_1' in cde5c157572c2beb0bd959b68c61d4ee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:47,974 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/2d5edb4777a94743bd44c958b414ba86 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/2d5edb4777a94743bd44c958b414ba86 2024-12-05T11:56:47,981 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/B of cde5c157572c2beb0bd959b68c61d4ee into 2d5edb4777a94743bd44c958b414ba86(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:47,981 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:47,981 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/B, priority=12, startTime=1733399807485; duration=0sec 2024-12-05T11:56:47,981 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:47,981 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:B 2024-12-05T11:56:47,981 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:56:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:56:47,983 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:47,984 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:47,984 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:56:47,984 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#A#compaction#123 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:47,984 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/C is initiating minor compaction (all files) 2024-12-05T11:56:47,984 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/C in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:47,985 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/5ed68ff89c2e4c20811aa98f8779e10a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c463e01d23484ec098499b01c66d718b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c0f4669b1e154c2f8a20fc3244b6bc0e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/7de4d61bbcff4b139e7cc31ef61977ad] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=46.9 K 2024-12-05T11:56:47,985 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/bb185a7051cb42428ae74962e7b99ad1 is 175, key is test_row_0/A:col10/1733399805770/Put/seqid=0 2024-12-05T11:56:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-05T11:56:47,986 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed68ff89c2e4c20811aa98f8779e10a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733399803828 2024-12-05T11:56:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:56:47,987 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c463e01d23484ec098499b01c66d718b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733399803881 2024-12-05T11:56:47,987 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:47,988 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c0f4669b1e154c2f8a20fc3244b6bc0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399804555 2024-12-05T11:56:47,988 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:47,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:47,989 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 7de4d61bbcff4b139e7cc31ef61977ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733399805764 2024-12-05T11:56:47,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741972_1148 (size=31093) 2024-12-05T11:56:48,003 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#C#compaction#127 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:48,004 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/a6ab94c1ab5649bfa3b104d7cb9de7e1 is 50, key is test_row_0/C:col10/1733399805770/Put/seqid=0 2024-12-05T11:56:48,005 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/bb185a7051cb42428ae74962e7b99ad1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/bb185a7051cb42428ae74962e7b99ad1 2024-12-05T11:56:48,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741973_1149 (size=12139) 2024-12-05T11:56:48,013 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/A of cde5c157572c2beb0bd959b68c61d4ee into bb185a7051cb42428ae74962e7b99ad1(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:48,013 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:48,013 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/A, priority=12, startTime=1733399807485; duration=0sec 2024-12-05T11:56:48,013 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:48,013 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:A 2024-12-05T11:56:48,021 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/a6ab94c1ab5649bfa3b104d7cb9de7e1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a6ab94c1ab5649bfa3b104d7cb9de7e1 2024-12-05T11:56:48,027 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/C of cde5c157572c2beb0bd959b68c61d4ee into a6ab94c1ab5649bfa3b104d7cb9de7e1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:48,027 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:48,027 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/C, priority=12, startTime=1733399807485; duration=0sec 2024-12-05T11:56:48,027 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:48,027 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:C 2024-12-05T11:56:48,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/e173cbf94fcf415ea04eab72806ae1e6 2024-12-05T11:56:48,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/5b937007797e4876ad784fc380822bd4 is 50, key is test_row_0/C:col10/1733399807551/Put/seqid=0 2024-12-05T11:56:48,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741974_1150 (size=12001) 2024-12-05T11:56:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:56:48,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T11:56:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:48,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:48,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:48,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399868258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:48,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399868258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:48,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T11:56:48,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:48,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:48,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:48,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:48,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:56:48,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T11:56:48,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:48,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:48,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/5b937007797e4876ad784fc380822bd4 2024-12-05T11:56:48,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/46b598e910f1442b874912f2ee77fa10 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/46b598e910f1442b874912f2ee77fa10 2024-12-05T11:56:48,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/46b598e910f1442b874912f2ee77fa10, entries=200, sequenceid=93, filesize=38.6 K 2024-12-05T11:56:48,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/e173cbf94fcf415ea04eab72806ae1e6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/e173cbf94fcf415ea04eab72806ae1e6 2024-12-05T11:56:48,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/e173cbf94fcf415ea04eab72806ae1e6, entries=150, sequenceid=93, filesize=11.7 K 2024-12-05T11:56:48,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/5b937007797e4876ad784fc380822bd4 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/5b937007797e4876ad784fc380822bd4 2024-12-05T11:56:48,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/5b937007797e4876ad784fc380822bd4, entries=150, sequenceid=93, filesize=11.7 K 2024-12-05T11:56:48,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for cde5c157572c2beb0bd959b68c61d4ee in 931ms, sequenceid=93, compaction requested=false 2024-12-05T11:56:48,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:48,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T11:56:48,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:48,601 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T11:56:48,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205bcae8ef5c8384c1bb2cabc985190edf7_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399807622/Put/seqid=0 2024-12-05T11:56:48,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741975_1151 (size=12154) 2024-12-05T11:56:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:56:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:48,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:48,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399868778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:48,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399868783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:48,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:48,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:48,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:48,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399868888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:48,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399868888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:48,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:48,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:48,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:49,021 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205bcae8ef5c8384c1bb2cabc985190edf7_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bcae8ef5c8384c1bb2cabc985190edf7_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:49,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/a091559db89646179efca5dd6e554a3d, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:49,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/a091559db89646179efca5dd6e554a3d is 175, key is test_row_0/A:col10/1733399807622/Put/seqid=0 2024-12-05T11:56:49,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741976_1152 (size=30955) 2024-12-05T11:56:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399869098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399869098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:49,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:56:49,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:49,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:49,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399869408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:49,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399869408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:49,430 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/a091559db89646179efca5dd6e554a3d 2024-12-05T11:56:49,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/615aab50d7bc4036ba62277df8f1608e is 50, key is test_row_0/B:col10/1733399807622/Put/seqid=0 2024-12-05T11:56:49,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741977_1153 (size=12001) 2024-12-05T11:56:49,843 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/615aab50d7bc4036ba62277df8f1608e 2024-12-05T11:56:49,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/f3403f0943e84c3ea88c8083ea3cbf81 is 50, key is test_row_0/C:col10/1733399807622/Put/seqid=0 2024-12-05T11:56:49,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741978_1154 (size=12001) 2024-12-05T11:56:49,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399869918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:49,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399869918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:49,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399869947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:49,948 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in cde5c157572c2beb0bd959b68c61d4ee of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:49,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399869958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:49,960 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in cde5c157572c2beb0bd959b68c61d4ee of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:49,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399869968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:49,970 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_1' in cde5c157572c2beb0bd959b68c61d4ee of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:56:50,255 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/f3403f0943e84c3ea88c8083ea3cbf81 2024-12-05T11:56:50,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/a091559db89646179efca5dd6e554a3d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/a091559db89646179efca5dd6e554a3d 2024-12-05T11:56:50,264 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/a091559db89646179efca5dd6e554a3d, entries=150, sequenceid=121, filesize=30.2 K 2024-12-05T11:56:50,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/615aab50d7bc4036ba62277df8f1608e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/615aab50d7bc4036ba62277df8f1608e 2024-12-05T11:56:50,270 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/615aab50d7bc4036ba62277df8f1608e, entries=150, sequenceid=121, filesize=11.7 K 2024-12-05T11:56:50,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/f3403f0943e84c3ea88c8083ea3cbf81 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/f3403f0943e84c3ea88c8083ea3cbf81 2024-12-05T11:56:50,275 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/f3403f0943e84c3ea88c8083ea3cbf81, entries=150, sequenceid=121, filesize=11.7 K 2024-12-05T11:56:50,276 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for cde5c157572c2beb0bd959b68c61d4ee in 1675ms, sequenceid=121, compaction requested=true 2024-12-05T11:56:50,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:50,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:50,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-05T11:56:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-05T11:56:50,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-05T11:56:50,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2900 sec 2024-12-05T11:56:50,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.2960 sec 2024-12-05T11:56:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:50,931 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:50,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T11:56:50,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205952cd0bf0e4c466f80d3a94f055c724e_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399810930/Put/seqid=0 2024-12-05T11:56:50,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741979_1155 (size=17284) 2024-12-05T11:56:50,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399870980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:50,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399870985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:51,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399871088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:51,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:51,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:51,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:51,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399871098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:51,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399871297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:51,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399871308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:51,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:51,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:51,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:51,349 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:51,362 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205952cd0bf0e4c466f80d3a94f055c724e_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205952cd0bf0e4c466f80d3a94f055c724e_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:51,363 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/90e6e771a1a2463294f1cef086eea194, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:51,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/90e6e771a1a2463294f1cef086eea194 is 175, key is test_row_0/A:col10/1733399810930/Put/seqid=0 2024-12-05T11:56:51,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741980_1156 (size=48389) 2024-12-05T11:56:51,369 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/90e6e771a1a2463294f1cef086eea194 2024-12-05T11:56:51,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/0a3dc046df944145a9c452303fa21108 is 50, key is test_row_0/B:col10/1733399810930/Put/seqid=0 2024-12-05T11:56:51,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741981_1157 (size=12151) 2024-12-05T11:56:51,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399871608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:51,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399871618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:51,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/0a3dc046df944145a9c452303fa21108 2024-12-05T11:56:51,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/fa6bd63cfffc493992b11638a82b6f5a is 50, key is test_row_0/C:col10/1733399810930/Put/seqid=0 2024-12-05T11:56:51,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741982_1158 (size=12151) 2024-12-05T11:56:52,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399872118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:52,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399872128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:52,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:52,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:56:52,158 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:52,158 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-05T11:56:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T11:56:52,160 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:52,160 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:52,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:52,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/fa6bd63cfffc493992b11638a82b6f5a 2024-12-05T11:56:52,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/90e6e771a1a2463294f1cef086eea194 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/90e6e771a1a2463294f1cef086eea194 2024-12-05T11:56:52,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/90e6e771a1a2463294f1cef086eea194, entries=250, sequenceid=134, filesize=47.3 K 2024-12-05T11:56:52,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/0a3dc046df944145a9c452303fa21108 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0a3dc046df944145a9c452303fa21108 2024-12-05T11:56:52,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0a3dc046df944145a9c452303fa21108, entries=150, sequenceid=134, filesize=11.9 K 2024-12-05T11:56:52,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/fa6bd63cfffc493992b11638a82b6f5a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/fa6bd63cfffc493992b11638a82b6f5a 2024-12-05T11:56:52,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/fa6bd63cfffc493992b11638a82b6f5a, entries=150, sequenceid=134, filesize=11.9 K 2024-12-05T11:56:52,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for cde5c157572c2beb0bd959b68c61d4ee in 1313ms, sequenceid=134, compaction requested=true 2024-12-05T11:56:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:52,244 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:56:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:52,245 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:56:52,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:52,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:52,246 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149986 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:56:52,246 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:56:52,246 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/A is initiating minor compaction (all files) 2024-12-05T11:56:52,246 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/A in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:52,246 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/bb185a7051cb42428ae74962e7b99ad1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/46b598e910f1442b874912f2ee77fa10, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/a091559db89646179efca5dd6e554a3d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/90e6e771a1a2463294f1cef086eea194] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=146.5 K 2024-12-05T11:56:52,246 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:52,246 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/bb185a7051cb42428ae74962e7b99ad1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/46b598e910f1442b874912f2ee77fa10, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/a091559db89646179efca5dd6e554a3d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/90e6e771a1a2463294f1cef086eea194] 2024-12-05T11:56:52,246 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/B is initiating minor compaction (all files) 2024-12-05T11:56:52,246 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/B in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:52,247 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/2d5edb4777a94743bd44c958b414ba86, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/e173cbf94fcf415ea04eab72806ae1e6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/615aab50d7bc4036ba62277df8f1608e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0a3dc046df944145a9c452303fa21108] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=47.2 K 2024-12-05T11:56:52,247 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb185a7051cb42428ae74962e7b99ad1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733399805764 2024-12-05T11:56:52,247 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46b598e910f1442b874912f2ee77fa10, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733399806912 2024-12-05T11:56:52,247 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d5edb4777a94743bd44c958b414ba86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733399805764 2024-12-05T11:56:52,248 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a091559db89646179efca5dd6e554a3d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733399807594 2024-12-05T11:56:52,248 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting e173cbf94fcf415ea04eab72806ae1e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733399807551 2024-12-05T11:56:52,248 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90e6e771a1a2463294f1cef086eea194, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399808776 2024-12-05T11:56:52,248 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 615aab50d7bc4036ba62277df8f1608e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733399807594 2024-12-05T11:56:52,248 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a3dc046df944145a9c452303fa21108, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399808776 2024-12-05T11:56:52,254 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:52,256 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#B#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:52,257 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/1b1db4a438bb44588d769879d9edff9d is 50, key is test_row_0/B:col10/1733399810930/Put/seqid=0 2024-12-05T11:56:52,258 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205e2e33be792eb4ba9b3f910db2b6f5e9b_cde5c157572c2beb0bd959b68c61d4ee store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:52,260 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205e2e33be792eb4ba9b3f910db2b6f5e9b_cde5c157572c2beb0bd959b68c61d4ee, store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:52,260 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e2e33be792eb4ba9b3f910db2b6f5e9b_cde5c157572c2beb0bd959b68c61d4ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T11:56:52,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741983_1159 (size=12425) 2024-12-05T11:56:52,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741984_1160 (size=4469) 2024-12-05T11:56:52,276 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/1b1db4a438bb44588d769879d9edff9d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/1b1db4a438bb44588d769879d9edff9d 2024-12-05T11:56:52,282 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/B of cde5c157572c2beb0bd959b68c61d4ee into 1b1db4a438bb44588d769879d9edff9d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:52,282 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:52,282 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/B, priority=12, startTime=1733399812244; duration=0sec 2024-12-05T11:56:52,282 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:52,282 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:B 2024-12-05T11:56:52,282 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:56:52,285 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:56:52,285 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/C is initiating minor compaction (all files) 2024-12-05T11:56:52,285 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/C in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:52,285 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a6ab94c1ab5649bfa3b104d7cb9de7e1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/5b937007797e4876ad784fc380822bd4, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/f3403f0943e84c3ea88c8083ea3cbf81, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/fa6bd63cfffc493992b11638a82b6f5a] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=47.2 K 2024-12-05T11:56:52,285 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting a6ab94c1ab5649bfa3b104d7cb9de7e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733399805764 2024-12-05T11:56:52,286 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b937007797e4876ad784fc380822bd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733399807551 2024-12-05T11:56:52,286 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting f3403f0943e84c3ea88c8083ea3cbf81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733399807594 2024-12-05T11:56:52,286 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting fa6bd63cfffc493992b11638a82b6f5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399808776 2024-12-05T11:56:52,295 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#C#compaction#137 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:52,296 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/792a8a04a3a34e21922e384aaa45405b is 50, key is test_row_0/C:col10/1733399810930/Put/seqid=0 2024-12-05T11:56:52,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741985_1161 (size=12425) 2024-12-05T11:56:52,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T11:56:52,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:52,313 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T11:56:52,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205036c26a0b0d4471e98a95f23b1caea51_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399810984/Put/seqid=0 2024-12-05T11:56:52,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741986_1162 (size=12304) 2024-12-05T11:56:52,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:52,335 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205036c26a0b0d4471e98a95f23b1caea51_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205036c26a0b0d4471e98a95f23b1caea51_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:52,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2d447913cfe04cb5ba6d44c729334447, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:52,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2d447913cfe04cb5ba6d44c729334447 is 175, key is test_row_0/A:col10/1733399810984/Put/seqid=0 2024-12-05T11:56:52,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741987_1163 (size=31105) 2024-12-05T11:56:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T11:56:52,672 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#A#compaction#135 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:52,673 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2ec971f80bf94ff1ae7d716f496b2574 is 175, key is test_row_0/A:col10/1733399810930/Put/seqid=0 2024-12-05T11:56:52,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741988_1164 (size=31379) 2024-12-05T11:56:52,707 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/792a8a04a3a34e21922e384aaa45405b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/792a8a04a3a34e21922e384aaa45405b 2024-12-05T11:56:52,713 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/C of cde5c157572c2beb0bd959b68c61d4ee into 792a8a04a3a34e21922e384aaa45405b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:52,713 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:52,713 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/C, priority=12, startTime=1733399812245; duration=0sec 2024-12-05T11:56:52,713 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:52,713 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:C 2024-12-05T11:56:52,752 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2d447913cfe04cb5ba6d44c729334447 2024-12-05T11:56:52,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/bb0c0f2d719c4090bbd5c0d287fb0edb is 50, key is test_row_0/B:col10/1733399810984/Put/seqid=0 2024-12-05T11:56:52,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741989_1165 (size=12151) 2024-12-05T11:56:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T11:56:53,085 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2ec971f80bf94ff1ae7d716f496b2574 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2ec971f80bf94ff1ae7d716f496b2574 2024-12-05T11:56:53,091 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/A of cde5c157572c2beb0bd959b68c61d4ee into 2ec971f80bf94ff1ae7d716f496b2574(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:53,091 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:53,091 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/A, priority=12, startTime=1733399812244; duration=0sec 2024-12-05T11:56:53,091 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:53,091 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:A 2024-12-05T11:56:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:53,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:53,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399873151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:53,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399873152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:53,186 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/bb0c0f2d719c4090bbd5c0d287fb0edb 2024-12-05T11:56:53,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/a559022980ea4efda6ac975a73c856a2 is 50, key is test_row_0/C:col10/1733399810984/Put/seqid=0 2024-12-05T11:56:53,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741990_1166 (size=12151) 2024-12-05T11:56:53,206 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/a559022980ea4efda6ac975a73c856a2 2024-12-05T11:56:53,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2d447913cfe04cb5ba6d44c729334447 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2d447913cfe04cb5ba6d44c729334447 2024-12-05T11:56:53,216 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2d447913cfe04cb5ba6d44c729334447, entries=150, sequenceid=158, filesize=30.4 K 2024-12-05T11:56:53,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/bb0c0f2d719c4090bbd5c0d287fb0edb as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bb0c0f2d719c4090bbd5c0d287fb0edb 2024-12-05T11:56:53,222 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bb0c0f2d719c4090bbd5c0d287fb0edb, entries=150, sequenceid=158, filesize=11.9 K 2024-12-05T11:56:53,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/a559022980ea4efda6ac975a73c856a2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a559022980ea4efda6ac975a73c856a2 2024-12-05T11:56:53,230 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a559022980ea4efda6ac975a73c856a2, entries=150, sequenceid=158, filesize=11.9 K 2024-12-05T11:56:53,231 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for cde5c157572c2beb0bd959b68c61d4ee in 918ms, sequenceid=158, compaction requested=false 2024-12-05T11:56:53,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:53,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:53,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-05T11:56:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-05T11:56:53,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-05T11:56:53,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0740 sec 2024-12-05T11:56:53,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.0780 sec 2024-12-05T11:56:53,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:53,260 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:53,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T11:56:53,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055eeac40d22fd4d0da2d7448c4707b42d_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399813149/Put/seqid=0 2024-12-05T11:56:53,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741991_1167 (size=14794) 2024-12-05T11:56:53,283 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:53,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055eeac40d22fd4d0da2d7448c4707b42d_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055eeac40d22fd4d0da2d7448c4707b42d_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:53,291 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2171ede5a71049118a4d2ed5ff5275d2, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:53,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2171ede5a71049118a4d2ed5ff5275d2 is 175, key is test_row_0/A:col10/1733399813149/Put/seqid=0 2024-12-05T11:56:53,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741992_1168 (size=39749) 2024-12-05T11:56:53,296 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2171ede5a71049118a4d2ed5ff5275d2 2024-12-05T11:56:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T11:56:53,298 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:53,299 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-05T11:56:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:56:53,301 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:53,301 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:53,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:53,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399873305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:53,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399873306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:53,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:53,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/0288a714be904057a2d171ff2f7531fd is 50, key is test_row_0/B:col10/1733399813149/Put/seqid=0 2024-12-05T11:56:53,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741993_1169 (size=12151) 2024-12-05T11:56:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:56:53,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:53,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399873418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399873418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:53,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T11:56:53,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:53,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:53,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:53,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:53,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:53,533 INFO [master/80666a11a09f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T11:56:53,533 INFO [master/80666a11a09f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T11:56:53,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T11:56:53,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:53,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:53,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:53,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:53,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:56:53,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399873628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:53,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399873628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:53,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/0288a714be904057a2d171ff2f7531fd 2024-12-05T11:56:53,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/7d6567bbc13543e29de90912a5b0d7c1 is 50, key is test_row_0/C:col10/1733399813149/Put/seqid=0 2024-12-05T11:56:53,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741994_1170 (size=12151) 2024-12-05T11:56:53,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T11:56:53,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:53,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:53,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:53,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:53,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:53,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T11:56:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:53,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:56:53,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399873938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:53,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399873938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:53,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399873957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:53,959 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in cde5c157572c2beb0bd959b68c61d4ee of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:53,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399873998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:53,999 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in cde5c157572c2beb0bd959b68c61d4ee of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:54,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399874008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:54,009 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_1' in cde5c157572c2beb0bd959b68c61d4ee of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:56:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:54,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T11:56:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:54,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:54,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/7d6567bbc13543e29de90912a5b0d7c1 2024-12-05T11:56:54,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2171ede5a71049118a4d2ed5ff5275d2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2171ede5a71049118a4d2ed5ff5275d2 2024-12-05T11:56:54,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2171ede5a71049118a4d2ed5ff5275d2, entries=200, sequenceid=175, filesize=38.8 K 2024-12-05T11:56:54,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/0288a714be904057a2d171ff2f7531fd as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0288a714be904057a2d171ff2f7531fd 2024-12-05T11:56:54,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0288a714be904057a2d171ff2f7531fd, entries=150, sequenceid=175, filesize=11.9 K 2024-12-05T11:56:54,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/7d6567bbc13543e29de90912a5b0d7c1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/7d6567bbc13543e29de90912a5b0d7c1 2024-12-05T11:56:54,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/7d6567bbc13543e29de90912a5b0d7c1, entries=150, sequenceid=175, filesize=11.9 K 2024-12-05T11:56:54,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for cde5c157572c2beb0bd959b68c61d4ee in 890ms, sequenceid=175, compaction requested=true 2024-12-05T11:56:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:54,151 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:54,151 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:54,152 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:54,152 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/B is initiating minor compaction (all files) 2024-12-05T11:56:54,152 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/B in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:54,152 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/1b1db4a438bb44588d769879d9edff9d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bb0c0f2d719c4090bbd5c0d287fb0edb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0288a714be904057a2d171ff2f7531fd] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=35.9 K 2024-12-05T11:56:54,152 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:54,152 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/A is initiating minor compaction (all files) 2024-12-05T11:56:54,153 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/A in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:54,153 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b1db4a438bb44588d769879d9edff9d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399808776 2024-12-05T11:56:54,153 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2ec971f80bf94ff1ae7d716f496b2574, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2d447913cfe04cb5ba6d44c729334447, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2171ede5a71049118a4d2ed5ff5275d2] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=99.8 K 2024-12-05T11:56:54,153 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:54,153 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2ec971f80bf94ff1ae7d716f496b2574, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2d447913cfe04cb5ba6d44c729334447, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2171ede5a71049118a4d2ed5ff5275d2] 2024-12-05T11:56:54,153 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting bb0c0f2d719c4090bbd5c0d287fb0edb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733399810963 2024-12-05T11:56:54,153 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ec971f80bf94ff1ae7d716f496b2574, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399808776 2024-12-05T11:56:54,153 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d447913cfe04cb5ba6d44c729334447, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733399810963 2024-12-05T11:56:54,154 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0288a714be904057a2d171ff2f7531fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733399813148 2024-12-05T11:56:54,154 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2171ede5a71049118a4d2ed5ff5275d2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733399813148 2024-12-05T11:56:54,160 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#B#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:54,160 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/c0d45fcba7f84f0cacc235c13f7285d0 is 50, key is test_row_0/B:col10/1733399813149/Put/seqid=0 2024-12-05T11:56:54,161 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:54,165 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412055f79d60d1dad48299df338bd86dda4a8_cde5c157572c2beb0bd959b68c61d4ee store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:54,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741995_1171 (size=12527) 2024-12-05T11:56:54,167 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412055f79d60d1dad48299df338bd86dda4a8_cde5c157572c2beb0bd959b68c61d4ee, store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:54,167 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055f79d60d1dad48299df338bd86dda4a8_cde5c157572c2beb0bd959b68c61d4ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:54,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741996_1172 (size=4469) 2024-12-05T11:56:54,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T11:56:54,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:54,222 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T11:56:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205804d6bbb7e614e0090d7b4a7830800b4_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399813302/Put/seqid=0 2024-12-05T11:56:54,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741997_1173 (size=12304) 2024-12-05T11:56:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:56:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:54,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:54,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399874469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:54,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399874472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:54,572 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/c0d45fcba7f84f0cacc235c13f7285d0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/c0d45fcba7f84f0cacc235c13f7285d0 2024-12-05T11:56:54,572 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#A#compaction#145 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:54,573 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/0fb18a9ad1454385a2f7e684eb535aca is 175, key is test_row_0/A:col10/1733399813149/Put/seqid=0 2024-12-05T11:56:54,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741998_1174 (size=31481) 2024-12-05T11:56:54,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399874578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:54,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399874578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:54,581 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/B of cde5c157572c2beb0bd959b68c61d4ee into c0d45fcba7f84f0cacc235c13f7285d0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:54,581 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:54,581 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/B, priority=13, startTime=1733399814151; duration=0sec 2024-12-05T11:56:54,581 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:54,581 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:B 2024-12-05T11:56:54,581 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:54,582 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:54,582 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/C is initiating minor compaction (all files) 2024-12-05T11:56:54,582 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/C in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:54,582 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/792a8a04a3a34e21922e384aaa45405b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a559022980ea4efda6ac975a73c856a2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/7d6567bbc13543e29de90912a5b0d7c1] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=35.9 K 2024-12-05T11:56:54,583 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 792a8a04a3a34e21922e384aaa45405b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399808776 2024-12-05T11:56:54,583 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting a559022980ea4efda6ac975a73c856a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733399810963 2024-12-05T11:56:54,583 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d6567bbc13543e29de90912a5b0d7c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733399813148 2024-12-05T11:56:54,589 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#C#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:54,589 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/38ebf032d2a74783afb132ea15ccfc30 is 50, key is test_row_0/C:col10/1733399813149/Put/seqid=0 2024-12-05T11:56:54,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741999_1175 (size=12527) 2024-12-05T11:56:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:54,637 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205804d6bbb7e614e0090d7b4a7830800b4_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205804d6bbb7e614e0090d7b4a7830800b4_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/51f646340b42491e9ad14f9e0f7bd094, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/51f646340b42491e9ad14f9e0f7bd094 is 175, key is test_row_0/A:col10/1733399813302/Put/seqid=0 2024-12-05T11:56:54,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742000_1176 (size=31105) 2024-12-05T11:56:54,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399874787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:54,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399874787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:54,983 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/0fb18a9ad1454385a2f7e684eb535aca as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0fb18a9ad1454385a2f7e684eb535aca 2024-12-05T11:56:54,988 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/A of cde5c157572c2beb0bd959b68c61d4ee into 0fb18a9ad1454385a2f7e684eb535aca(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:54,988 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:54,988 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/A, priority=13, startTime=1733399814151; duration=0sec 2024-12-05T11:56:54,988 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:54,988 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:A 2024-12-05T11:56:55,003 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/38ebf032d2a74783afb132ea15ccfc30 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/38ebf032d2a74783afb132ea15ccfc30 2024-12-05T11:56:55,009 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/C of cde5c157572c2beb0bd959b68c61d4ee into 38ebf032d2a74783afb132ea15ccfc30(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:55,009 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:55,009 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/C, priority=13, startTime=1733399814151; duration=0sec 2024-12-05T11:56:55,009 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:55,009 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:C 2024-12-05T11:56:55,042 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/51f646340b42491e9ad14f9e0f7bd094 2024-12-05T11:56:55,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/bac44bcbf0584eb28e8c51a188ba920a is 50, key is test_row_0/B:col10/1733399813302/Put/seqid=0 2024-12-05T11:56:55,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742001_1177 (size=12151) 2024-12-05T11:56:55,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399875098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:55,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399875098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:56:55,455 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/bac44bcbf0584eb28e8c51a188ba920a 2024-12-05T11:56:55,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/b54ffb0492434be9a7072bcdb7981e16 is 50, key is test_row_0/C:col10/1733399813302/Put/seqid=0 2024-12-05T11:56:55,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742002_1178 (size=12151) 2024-12-05T11:56:55,477 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/b54ffb0492434be9a7072bcdb7981e16 2024-12-05T11:56:55,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/51f646340b42491e9ad14f9e0f7bd094 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/51f646340b42491e9ad14f9e0f7bd094 2024-12-05T11:56:55,487 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/51f646340b42491e9ad14f9e0f7bd094, entries=150, sequenceid=196, filesize=30.4 K 2024-12-05T11:56:55,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/bac44bcbf0584eb28e8c51a188ba920a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bac44bcbf0584eb28e8c51a188ba920a 2024-12-05T11:56:55,492 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bac44bcbf0584eb28e8c51a188ba920a, entries=150, sequenceid=196, filesize=11.9 K 2024-12-05T11:56:55,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/b54ffb0492434be9a7072bcdb7981e16 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/b54ffb0492434be9a7072bcdb7981e16 2024-12-05T11:56:55,497 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/b54ffb0492434be9a7072bcdb7981e16, entries=150, sequenceid=196, filesize=11.9 K 2024-12-05T11:56:55,498 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for cde5c157572c2beb0bd959b68c61d4ee in 1276ms, sequenceid=196, compaction requested=false 2024-12-05T11:56:55,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:55,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:55,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-05T11:56:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-05T11:56:55,503 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-05T11:56:55,503 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1980 sec 2024-12-05T11:56:55,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.2050 sec 2024-12-05T11:56:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:55,610 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:55,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-05T11:56:55,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ff52a6cad26b45729ad35eca13724848_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399814470/Put/seqid=0 2024-12-05T11:56:55,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742003_1179 (size=17284) 2024-12-05T11:56:55,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399875649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:55,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399875652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:55,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399875758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:55,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399875758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:55,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399875968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:55,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399875968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:56,032 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:56,036 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ff52a6cad26b45729ad35eca13724848_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ff52a6cad26b45729ad35eca13724848_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:56,037 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/11874f1a44fe4f658a72bc8923c9c767, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:56,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/11874f1a44fe4f658a72bc8923c9c767 is 175, key is test_row_0/A:col10/1733399814470/Put/seqid=0 2024-12-05T11:56:56,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742004_1180 (size=48389) 2024-12-05T11:56:56,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:56,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399876278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399876278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:56,442 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/11874f1a44fe4f658a72bc8923c9c767 2024-12-05T11:56:56,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/0dbf79e815e843d88230118e7fce60ac is 50, key is test_row_0/B:col10/1733399814470/Put/seqid=0 2024-12-05T11:56:56,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742005_1181 (size=12151) 2024-12-05T11:56:56,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399876788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:56,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399876788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:56,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/0dbf79e815e843d88230118e7fce60ac 2024-12-05T11:56:56,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/e9cb2ae6631e41489878a3aaf5dbceee is 50, key is test_row_0/C:col10/1733399814470/Put/seqid=0 2024-12-05T11:56:56,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742006_1182 (size=12151) 2024-12-05T11:56:57,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/e9cb2ae6631e41489878a3aaf5dbceee 2024-12-05T11:56:57,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/11874f1a44fe4f658a72bc8923c9c767 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/11874f1a44fe4f658a72bc8923c9c767 2024-12-05T11:56:57,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/11874f1a44fe4f658a72bc8923c9c767, entries=250, sequenceid=216, filesize=47.3 K 2024-12-05T11:56:57,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/0dbf79e815e843d88230118e7fce60ac as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0dbf79e815e843d88230118e7fce60ac 2024-12-05T11:56:57,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0dbf79e815e843d88230118e7fce60ac, entries=150, sequenceid=216, filesize=11.9 K 2024-12-05T11:56:57,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/e9cb2ae6631e41489878a3aaf5dbceee as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/e9cb2ae6631e41489878a3aaf5dbceee 2024-12-05T11:56:57,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/e9cb2ae6631e41489878a3aaf5dbceee, entries=150, sequenceid=216, filesize=11.9 K 2024-12-05T11:56:57,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for cde5c157572c2beb0bd959b68c61d4ee in 1679ms, sequenceid=216, compaction requested=true 2024-12-05T11:56:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:57,289 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:57,289 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:57,290 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:57,290 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:57,290 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/A is initiating minor compaction (all files) 2024-12-05T11:56:57,290 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/B is initiating minor compaction (all files) 2024-12-05T11:56:57,290 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/A in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:57,290 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/B in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:57,290 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0fb18a9ad1454385a2f7e684eb535aca, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/51f646340b42491e9ad14f9e0f7bd094, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/11874f1a44fe4f658a72bc8923c9c767] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=108.4 K 2024-12-05T11:56:57,290 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/c0d45fcba7f84f0cacc235c13f7285d0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bac44bcbf0584eb28e8c51a188ba920a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0dbf79e815e843d88230118e7fce60ac] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=36.0 K 2024-12-05T11:56:57,290 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:57,290 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0fb18a9ad1454385a2f7e684eb535aca, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/51f646340b42491e9ad14f9e0f7bd094, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/11874f1a44fe4f658a72bc8923c9c767] 2024-12-05T11:56:57,291 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d45fcba7f84f0cacc235c13f7285d0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733399813148 2024-12-05T11:56:57,291 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fb18a9ad1454385a2f7e684eb535aca, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733399813148 2024-12-05T11:56:57,291 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting bac44bcbf0584eb28e8c51a188ba920a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733399813286 2024-12-05T11:56:57,291 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51f646340b42491e9ad14f9e0f7bd094, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733399813286 2024-12-05T11:56:57,291 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dbf79e815e843d88230118e7fce60ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733399814470 2024-12-05T11:56:57,292 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11874f1a44fe4f658a72bc8923c9c767, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733399814467 2024-12-05T11:56:57,299 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#B#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:57,299 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/4730085af93947be89fde2f2cbfba978 is 50, key is test_row_0/B:col10/1733399814470/Put/seqid=0 2024-12-05T11:56:57,300 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:57,304 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205f5058c7bdc1940cc9dce3619855a0cf8_cde5c157572c2beb0bd959b68c61d4ee store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:57,305 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205f5058c7bdc1940cc9dce3619855a0cf8_cde5c157572c2beb0bd959b68c61d4ee, store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:57,306 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f5058c7bdc1940cc9dce3619855a0cf8_cde5c157572c2beb0bd959b68c61d4ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:57,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742007_1183 (size=12629) 2024-12-05T11:56:57,314 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/4730085af93947be89fde2f2cbfba978 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/4730085af93947be89fde2f2cbfba978 2024-12-05T11:56:57,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742008_1184 (size=4469) 2024-12-05T11:56:57,319 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/B of cde5c157572c2beb0bd959b68c61d4ee into 4730085af93947be89fde2f2cbfba978(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:57,319 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:57,319 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/B, priority=13, startTime=1733399817289; duration=0sec 2024-12-05T11:56:57,319 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:57,319 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:B 2024-12-05T11:56:57,319 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:57,321 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:57,321 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/C is initiating minor compaction (all files) 2024-12-05T11:56:57,321 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/C in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:57,321 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/38ebf032d2a74783afb132ea15ccfc30, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/b54ffb0492434be9a7072bcdb7981e16, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/e9cb2ae6631e41489878a3aaf5dbceee] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=36.0 K 2024-12-05T11:56:57,321 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 38ebf032d2a74783afb132ea15ccfc30, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733399813148 2024-12-05T11:56:57,322 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting b54ffb0492434be9a7072bcdb7981e16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733399813286 2024-12-05T11:56:57,322 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting e9cb2ae6631e41489878a3aaf5dbceee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733399814470 2024-12-05T11:56:57,333 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#C#compaction#155 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:57,333 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/eb7d45b4722a468e93586cc0d3b48001 is 50, key is test_row_0/C:col10/1733399814470/Put/seqid=0 2024-12-05T11:56:57,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742009_1185 (size=12629) 2024-12-05T11:56:57,366 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/eb7d45b4722a468e93586cc0d3b48001 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/eb7d45b4722a468e93586cc0d3b48001 2024-12-05T11:56:57,374 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/C of cde5c157572c2beb0bd959b68c61d4ee into eb7d45b4722a468e93586cc0d3b48001(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:57,374 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:57,374 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/C, priority=13, startTime=1733399817289; duration=0sec 2024-12-05T11:56:57,374 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:57,374 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:C 2024-12-05T11:56:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:56:57,468 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:57,468 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-05T11:56:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T11:56:57,471 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:57,472 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:57,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T11:56:57,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T11:56:57,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:57,625 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-05T11:56:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205de2b7ff34b70465b9fcf09bcbf684e23_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399815650/Put/seqid=0 2024-12-05T11:56:57,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742010_1186 (size=12304) 2024-12-05T11:56:57,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:57,661 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205de2b7ff34b70465b9fcf09bcbf684e23_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205de2b7ff34b70465b9fcf09bcbf684e23_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2014b9d1ee31458bb25bf95569553b35, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2014b9d1ee31458bb25bf95569553b35 is 175, key is test_row_0/A:col10/1733399815650/Put/seqid=0 2024-12-05T11:56:57,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742011_1187 (size=31105) 2024-12-05T11:56:57,716 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#A#compaction#154 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:57,716 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2625d69cf0374699a88123a2732c9885 is 175, key is test_row_0/A:col10/1733399814470/Put/seqid=0 2024-12-05T11:56:57,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742012_1188 (size=31583) 2024-12-05T11:56:57,725 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2625d69cf0374699a88123a2732c9885 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2625d69cf0374699a88123a2732c9885 2024-12-05T11:56:57,731 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/A of cde5c157572c2beb0bd959b68c61d4ee into 2625d69cf0374699a88123a2732c9885(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:57,731 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:57,731 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/A, priority=13, startTime=1733399817289; duration=0sec 2024-12-05T11:56:57,731 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:57,731 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:A 2024-12-05T11:56:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T11:56:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:57,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:57,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399877831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:57,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399877834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:57,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399877938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:57,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:57,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:57,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:57,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399877948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:58,070 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2014b9d1ee31458bb25bf95569553b35 2024-12-05T11:56:58,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/81c73b8ab48740fd9f82c213ce556c70 is 50, key is test_row_0/B:col10/1733399815650/Put/seqid=0 2024-12-05T11:56:58,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742013_1189 (size=12151) 2024-12-05T11:56:58,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T11:56:58,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399878147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:58,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399878158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:58,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399878458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:58,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:58,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:58,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:58,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399878468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:58,485 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/81c73b8ab48740fd9f82c213ce556c70 2024-12-05T11:56:58,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/486b9e68caef437e8986e50ef2c3f3fa is 50, key is test_row_0/C:col10/1733399815650/Put/seqid=0 2024-12-05T11:56:58,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742014_1190 (size=12151) 2024-12-05T11:56:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T11:56:58,903 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/486b9e68caef437e8986e50ef2c3f3fa 2024-12-05T11:56:58,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/2014b9d1ee31458bb25bf95569553b35 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2014b9d1ee31458bb25bf95569553b35 2024-12-05T11:56:58,912 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2014b9d1ee31458bb25bf95569553b35, entries=150, sequenceid=237, filesize=30.4 K 2024-12-05T11:56:58,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/81c73b8ab48740fd9f82c213ce556c70 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/81c73b8ab48740fd9f82c213ce556c70 2024-12-05T11:56:58,917 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/81c73b8ab48740fd9f82c213ce556c70, entries=150, sequenceid=237, filesize=11.9 K 2024-12-05T11:56:58,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/486b9e68caef437e8986e50ef2c3f3fa as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/486b9e68caef437e8986e50ef2c3f3fa 2024-12-05T11:56:58,922 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/486b9e68caef437e8986e50ef2c3f3fa, entries=150, sequenceid=237, filesize=11.9 K 2024-12-05T11:56:58,923 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for cde5c157572c2beb0bd959b68c61d4ee in 1298ms, sequenceid=237, compaction requested=false 2024-12-05T11:56:58,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:58,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:58,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-05T11:56:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-05T11:56:58,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-05T11:56:58,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4520 sec 2024-12-05T11:56:58,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4580 sec 2024-12-05T11:56:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:58,969 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:58,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-05T11:56:58,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ada3d6ce0efe480b8db58f873c5eb3f3_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399817830/Put/seqid=0 2024-12-05T11:56:58,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742015_1191 (size=14794) 2024-12-05T11:56:58,985 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:56:58,990 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ada3d6ce0efe480b8db58f873c5eb3f3_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ada3d6ce0efe480b8db58f873c5eb3f3_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:58,991 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/65dc4bb3c2fd48f7b30a657af92bb3d2, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:58,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/65dc4bb3c2fd48f7b30a657af92bb3d2 is 175, key is test_row_0/A:col10/1733399817830/Put/seqid=0 2024-12-05T11:56:58,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742016_1192 (size=39749) 2024-12-05T11:56:58,996 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/65dc4bb3c2fd48f7b30a657af92bb3d2 2024-12-05T11:56:59,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399878999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:59,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399879000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:59,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/b645ab0a06454444b72fc9bf21823858 is 50, key is test_row_0/B:col10/1733399817830/Put/seqid=0 2024-12-05T11:56:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:59,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742017_1193 (size=12151) 2024-12-05T11:56:59,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399879108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:59,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399879108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:59,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399879318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:59,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399879318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:59,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/b645ab0a06454444b72fc9bf21823858 2024-12-05T11:56:59,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/8a69b67220a94e2d8bf8c70537e6139c is 50, key is test_row_0/C:col10/1733399817830/Put/seqid=0 2024-12-05T11:56:59,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742018_1194 (size=12151) 2024-12-05T11:56:59,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/8a69b67220a94e2d8bf8c70537e6139c 2024-12-05T11:56:59,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/65dc4bb3c2fd48f7b30a657af92bb3d2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/65dc4bb3c2fd48f7b30a657af92bb3d2 2024-12-05T11:56:59,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/65dc4bb3c2fd48f7b30a657af92bb3d2, entries=200, sequenceid=256, filesize=38.8 K 2024-12-05T11:56:59,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/b645ab0a06454444b72fc9bf21823858 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/b645ab0a06454444b72fc9bf21823858 2024-12-05T11:56:59,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/b645ab0a06454444b72fc9bf21823858, entries=150, sequenceid=256, filesize=11.9 K 2024-12-05T11:56:59,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/8a69b67220a94e2d8bf8c70537e6139c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/8a69b67220a94e2d8bf8c70537e6139c 2024-12-05T11:56:59,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/8a69b67220a94e2d8bf8c70537e6139c, entries=150, sequenceid=256, filesize=11.9 K 2024-12-05T11:56:59,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for cde5c157572c2beb0bd959b68c61d4ee in 481ms, sequenceid=256, compaction requested=true 2024-12-05T11:56:59,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:59,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:56:59,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:59,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:56:59,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:59,450 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:59,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:56:59,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:59,450 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:59,451 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:59,451 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/A is initiating minor compaction (all files) 2024-12-05T11:56:59,451 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/A in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:59,451 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:59,452 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2625d69cf0374699a88123a2732c9885, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2014b9d1ee31458bb25bf95569553b35, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/65dc4bb3c2fd48f7b30a657af92bb3d2] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=100.0 K 2024-12-05T11:56:59,452 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/B is initiating minor compaction (all files) 2024-12-05T11:56:59,452 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/B in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:59,452 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:59,452 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2625d69cf0374699a88123a2732c9885, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2014b9d1ee31458bb25bf95569553b35, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/65dc4bb3c2fd48f7b30a657af92bb3d2] 2024-12-05T11:56:59,452 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/4730085af93947be89fde2f2cbfba978, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/81c73b8ab48740fd9f82c213ce556c70, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/b645ab0a06454444b72fc9bf21823858] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=36.1 K 2024-12-05T11:56:59,452 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2625d69cf0374699a88123a2732c9885, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733399814470 2024-12-05T11:56:59,452 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4730085af93947be89fde2f2cbfba978, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733399814470 2024-12-05T11:56:59,452 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2014b9d1ee31458bb25bf95569553b35, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733399815635 2024-12-05T11:56:59,452 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 81c73b8ab48740fd9f82c213ce556c70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733399815635 2024-12-05T11:56:59,453 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65dc4bb3c2fd48f7b30a657af92bb3d2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733399817827 2024-12-05T11:56:59,453 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting b645ab0a06454444b72fc9bf21823858, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733399817830 2024-12-05T11:56:59,459 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:59,459 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#B#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:59,460 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/03d703be4cb846589d6d72d72d2a5482 is 50, key is test_row_0/B:col10/1733399817830/Put/seqid=0 2024-12-05T11:56:59,464 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205ec24e8b94fde422daef6cadc82cd3a16_cde5c157572c2beb0bd959b68c61d4ee store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:59,464 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205ec24e8b94fde422daef6cadc82cd3a16_cde5c157572c2beb0bd959b68c61d4ee, store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:59,465 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ec24e8b94fde422daef6cadc82cd3a16_cde5c157572c2beb0bd959b68c61d4ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:56:59,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742019_1195 (size=12731) 2024-12-05T11:56:59,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742020_1196 (size=4469) 2024-12-05T11:56:59,487 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/03d703be4cb846589d6d72d72d2a5482 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/03d703be4cb846589d6d72d72d2a5482 2024-12-05T11:56:59,492 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/B of cde5c157572c2beb0bd959b68c61d4ee into 03d703be4cb846589d6d72d72d2a5482(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:59,492 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:59,492 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/B, priority=13, startTime=1733399819450; duration=0sec 2024-12-05T11:56:59,492 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:56:59,492 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:B 2024-12-05T11:56:59,492 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:56:59,493 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:56:59,493 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/C is initiating minor compaction (all files) 2024-12-05T11:56:59,493 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/C in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:59,493 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/eb7d45b4722a468e93586cc0d3b48001, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/486b9e68caef437e8986e50ef2c3f3fa, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/8a69b67220a94e2d8bf8c70537e6139c] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=36.1 K 2024-12-05T11:56:59,494 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting eb7d45b4722a468e93586cc0d3b48001, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733399814470 2024-12-05T11:56:59,494 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 486b9e68caef437e8986e50ef2c3f3fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733399815635 2024-12-05T11:56:59,494 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a69b67220a94e2d8bf8c70537e6139c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733399817830 2024-12-05T11:56:59,502 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#C#compaction#164 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:59,503 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/c008762ef7c749ccad2c7b3e581dcc42 is 50, key is test_row_0/C:col10/1733399817830/Put/seqid=0 2024-12-05T11:56:59,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742021_1197 (size=12731) 2024-12-05T11:56:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T11:56:59,618 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:56:59,619 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:56:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-05T11:56:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T11:56:59,620 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:56:59,621 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:56:59,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:56:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:56:59,628 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:56:59,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-05T11:56:59,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050df0572151504a60880e7df7a8dbaf01_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399818999/Put/seqid=0 2024-12-05T11:56:59,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742022_1198 (size=14994) 2024-12-05T11:56:59,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399879647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:59,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399879648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T11:56:59,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399879757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:59,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399879757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:59,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T11:56:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:59,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:59,872 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#A#compaction#163 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:56:59,873 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/c41df24679984ec3b424ffde4a1cd21d is 175, key is test_row_0/A:col10/1733399817830/Put/seqid=0 2024-12-05T11:56:59,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742023_1199 (size=31685) 2024-12-05T11:56:59,920 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/c008762ef7c749ccad2c7b3e581dcc42 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c008762ef7c749ccad2c7b3e581dcc42 2024-12-05T11:56:59,926 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/C of cde5c157572c2beb0bd959b68c61d4ee into c008762ef7c749ccad2c7b3e581dcc42(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:56:59,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T11:56:59,926 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:56:59,926 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/C, priority=13, startTime=1733399819450; duration=0sec 2024-12-05T11:56:59,926 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:56:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:59,926 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:C 2024-12-05T11:56:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:56:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:56:59,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:56:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T11:56:59,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399879967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:59,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:56:59,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:56:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399879967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:56:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:56:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:00,040 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:00,044 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050df0572151504a60880e7df7a8dbaf01_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050df0572151504a60880e7df7a8dbaf01_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:00,045 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/0e371a022fa548939c1d5eb8278e0903, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:57:00,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/0e371a022fa548939c1d5eb8278e0903 is 175, key is test_row_0/A:col10/1733399818999/Put/seqid=0 2024-12-05T11:57:00,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742024_1200 (size=39949) 2024-12-05T11:57:00,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T11:57:00,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:00,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:00,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:00,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T11:57:00,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:00,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:00,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:00,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T11:57:00,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399880278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:00,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399880278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:00,281 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/c41df24679984ec3b424ffde4a1cd21d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c41df24679984ec3b424ffde4a1cd21d 2024-12-05T11:57:00,286 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/A of cde5c157572c2beb0bd959b68c61d4ee into c41df24679984ec3b424ffde4a1cd21d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:00,286 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:57:00,286 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/A, priority=13, startTime=1733399819450; duration=0sec 2024-12-05T11:57:00,286 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:00,286 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:A 2024-12-05T11:57:00,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T11:57:00,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:00,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:00,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:00,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,450 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/0e371a022fa548939c1d5eb8278e0903 2024-12-05T11:57:00,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/8d676937f27049049a885b47fa831e78 is 50, key is test_row_0/B:col10/1733399818999/Put/seqid=0 2024-12-05T11:57:00,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742025_1201 (size=12301) 2024-12-05T11:57:00,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T11:57:00,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:00,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:00,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:00,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T11:57:00,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:00,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:00,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:00,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T11:57:00,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:00,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399880788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399880788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:00,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T11:57:00,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:00,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:00,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:00,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:00,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/8d676937f27049049a885b47fa831e78 2024-12-05T11:57:00,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/ab90c7418ef34889929f69f806157e8f is 50, key is test_row_0/C:col10/1733399818999/Put/seqid=0 2024-12-05T11:57:00,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742026_1202 (size=12301) 2024-12-05T11:57:01,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T11:57:01,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:01,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:01,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:01,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:01,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:01,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T11:57:01,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:01,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:01,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:01,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:01,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:01,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/ab90c7418ef34889929f69f806157e8f 2024-12-05T11:57:01,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/0e371a022fa548939c1d5eb8278e0903 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0e371a022fa548939c1d5eb8278e0903 2024-12-05T11:57:01,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0e371a022fa548939c1d5eb8278e0903, entries=200, sequenceid=277, filesize=39.0 K 2024-12-05T11:57:01,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/8d676937f27049049a885b47fa831e78 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/8d676937f27049049a885b47fa831e78 2024-12-05T11:57:01,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/8d676937f27049049a885b47fa831e78, entries=150, sequenceid=277, filesize=12.0 K 2024-12-05T11:57:01,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/ab90c7418ef34889929f69f806157e8f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/ab90c7418ef34889929f69f806157e8f 2024-12-05T11:57:01,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/ab90c7418ef34889929f69f806157e8f, entries=150, sequenceid=277, filesize=12.0 K 2024-12-05T11:57:01,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for cde5c157572c2beb0bd959b68c61d4ee in 1663ms, sequenceid=277, compaction requested=false 2024-12-05T11:57:01,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:57:01,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T11:57:01,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:01,306 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-05T11:57:01,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205855e6c32737d488891e8f36130af2737_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399819646/Put/seqid=0 2024-12-05T11:57:01,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742027_1203 (size=12454) 2024-12-05T11:57:01,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:01,322 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205855e6c32737d488891e8f36130af2737_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205855e6c32737d488891e8f36130af2737_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:01,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/c8ad4a9a170145d091581e701bb9721f, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:57:01,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/c8ad4a9a170145d091581e701bb9721f is 175, key is test_row_0/A:col10/1733399819646/Put/seqid=0 2024-12-05T11:57:01,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742028_1204 (size=31255) 2024-12-05T11:57:01,733 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/c8ad4a9a170145d091581e701bb9721f 2024-12-05T11:57:01,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/502c5c3e408242828dc1eebb029cbb17 is 50, key is test_row_0/B:col10/1733399819646/Put/seqid=0 2024-12-05T11:57:01,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742029_1205 (size=12301) 2024-12-05T11:57:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T11:57:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:01,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:01,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399881838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:01,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:01,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:01,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399881838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:01,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:01,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399881947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:01,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399881948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:02,144 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/502c5c3e408242828dc1eebb029cbb17 2024-12-05T11:57:02,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/a5f5d0fe89654173a871fc80018a573c is 50, key is test_row_0/C:col10/1733399819646/Put/seqid=0 2024-12-05T11:57:02,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399882158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:02,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399882158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:02,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742030_1206 (size=12301) 2024-12-05T11:57:02,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399882468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:02,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399882468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:02,563 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/a5f5d0fe89654173a871fc80018a573c 2024-12-05T11:57:02,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/c8ad4a9a170145d091581e701bb9721f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c8ad4a9a170145d091581e701bb9721f 2024-12-05T11:57:02,572 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c8ad4a9a170145d091581e701bb9721f, entries=150, sequenceid=295, filesize=30.5 K 2024-12-05T11:57:02,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/502c5c3e408242828dc1eebb029cbb17 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/502c5c3e408242828dc1eebb029cbb17 2024-12-05T11:57:02,577 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/502c5c3e408242828dc1eebb029cbb17, entries=150, sequenceid=295, filesize=12.0 K 2024-12-05T11:57:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/a5f5d0fe89654173a871fc80018a573c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a5f5d0fe89654173a871fc80018a573c 2024-12-05T11:57:02,582 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a5f5d0fe89654173a871fc80018a573c, entries=150, sequenceid=295, filesize=12.0 K 2024-12-05T11:57:02,583 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for cde5c157572c2beb0bd959b68c61d4ee in 1277ms, sequenceid=295, compaction requested=true 2024-12-05T11:57:02,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:57:02,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:02,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-05T11:57:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-05T11:57:02,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-05T11:57:02,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9630 sec 2024-12-05T11:57:02,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.9670 sec 2024-12-05T11:57:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:02,979 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:02,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T11:57:02,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058e8e6dfc5eb24649be9476ae4ef41196_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399821837/Put/seqid=0 2024-12-05T11:57:02,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742031_1207 (size=17534) 2024-12-05T11:57:02,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399882997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:03,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399883000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399883108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:03,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399883108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:03,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399883318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:03,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399883318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:03,393 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:03,397 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058e8e6dfc5eb24649be9476ae4ef41196_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058e8e6dfc5eb24649be9476ae4ef41196_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:03,398 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/6d3824ad2916464b936328837a443a2f, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:57:03,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/6d3824ad2916464b936328837a443a2f is 175, key is test_row_0/A:col10/1733399821837/Put/seqid=0 2024-12-05T11:57:03,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742032_1208 (size=48639) 2024-12-05T11:57:03,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399883628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:03,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399883628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T11:57:03,788 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:03,788 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-05T11:57:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T11:57:03,790 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:03,790 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:03,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:03,803 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/6d3824ad2916464b936328837a443a2f 2024-12-05T11:57:03,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/5eab5d3353ef42d69796fc2e0275ed2a is 50, key is test_row_0/B:col10/1733399821837/Put/seqid=0 2024-12-05T11:57:03,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742033_1209 (size=12301) 2024-12-05T11:57:03,820 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-12-05T11:57:03,820 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:03,820 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:03,820 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:03,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:03,820 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-12-05T11:57:03,820 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:03,820 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:03,820 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-12-05T11:57:03,820 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:03,820 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:03,820 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:03,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:03,820 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:03,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:03,821 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-12-05T11:57:03,821 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:03,821 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:03,821 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:03,821 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T11:57:03,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T11:57:03,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:03,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:03,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:03,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:03,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:03,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1733399884037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:04,038 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in cde5c157572c2beb0bd959b68c61d4ee of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:04,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46278 deadline: 1733399884058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:04,059 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_1' in cde5c157572c2beb0bd959b68c61d4ee of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:04,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T11:57:04,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:04,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T11:57:04,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46230 deadline: 1733399884108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:04,109 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in cde5c157572c2beb0bd959b68c61d4ee of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18335 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:04,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:04,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46258 deadline: 1733399884138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46276 deadline: 1733399884138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:04,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/5eab5d3353ef42d69796fc2e0275ed2a 2024-12-05T11:57:04,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/373f64e903df41d6902c4b7da0ee09cb is 50, key is test_row_0/C:col10/1733399821837/Put/seqid=0 2024-12-05T11:57:04,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742034_1210 (size=12301) 2024-12-05T11:57:04,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T11:57:04,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:04,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T11:57:04,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:04,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T11:57:04,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T11:57:04,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:04,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:04,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/373f64e903df41d6902c4b7da0ee09cb 2024-12-05T11:57:04,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/6d3824ad2916464b936328837a443a2f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6d3824ad2916464b936328837a443a2f 2024-12-05T11:57:04,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6d3824ad2916464b936328837a443a2f, entries=250, sequenceid=317, filesize=47.5 K 2024-12-05T11:57:04,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/5eab5d3353ef42d69796fc2e0275ed2a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/5eab5d3353ef42d69796fc2e0275ed2a 2024-12-05T11:57:04,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/5eab5d3353ef42d69796fc2e0275ed2a, entries=150, sequenceid=317, filesize=12.0 K 2024-12-05T11:57:04,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/373f64e903df41d6902c4b7da0ee09cb as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/373f64e903df41d6902c4b7da0ee09cb 2024-12-05T11:57:04,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/373f64e903df41d6902c4b7da0ee09cb, entries=150, sequenceid=317, filesize=12.0 K 2024-12-05T11:57:04,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for cde5c157572c2beb0bd959b68c61d4ee in 1679ms, sequenceid=317, compaction requested=true 2024-12-05T11:57:04,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:57:04,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:57:04,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:04,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:57:04,658 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:57:04,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:04,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cde5c157572c2beb0bd959b68c61d4ee:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:57:04,658 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:57:04,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:04,660 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151528 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:57:04,660 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:57:04,660 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/B is initiating minor compaction (all files) 2024-12-05T11:57:04,660 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/A is initiating minor compaction (all files) 2024-12-05T11:57:04,660 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/A in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,660 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/B in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,660 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c41df24679984ec3b424ffde4a1cd21d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0e371a022fa548939c1d5eb8278e0903, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c8ad4a9a170145d091581e701bb9721f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6d3824ad2916464b936328837a443a2f] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=148.0 K 2024-12-05T11:57:04,660 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/03d703be4cb846589d6d72d72d2a5482, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/8d676937f27049049a885b47fa831e78, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/502c5c3e408242828dc1eebb029cbb17, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/5eab5d3353ef42d69796fc2e0275ed2a] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=48.5 K 2024-12-05T11:57:04,660 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,660 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c41df24679984ec3b424ffde4a1cd21d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0e371a022fa548939c1d5eb8278e0903, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c8ad4a9a170145d091581e701bb9721f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6d3824ad2916464b936328837a443a2f] 2024-12-05T11:57:04,660 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 03d703be4cb846589d6d72d72d2a5482, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733399817830 2024-12-05T11:57:04,660 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c41df24679984ec3b424ffde4a1cd21d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733399817830 2024-12-05T11:57:04,660 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d676937f27049049a885b47fa831e78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733399818999 2024-12-05T11:57:04,660 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e371a022fa548939c1d5eb8278e0903, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733399818993 2024-12-05T11:57:04,661 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 502c5c3e408242828dc1eebb029cbb17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733399819643 2024-12-05T11:57:04,661 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8ad4a9a170145d091581e701bb9721f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733399819643 2024-12-05T11:57:04,661 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eab5d3353ef42d69796fc2e0275ed2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733399821837 2024-12-05T11:57:04,661 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d3824ad2916464b936328837a443a2f, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733399821831 2024-12-05T11:57:04,670 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#B#compaction#174 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:04,670 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/19f8310c9eb04900931768533a06d091 is 50, key is test_row_0/B:col10/1733399821837/Put/seqid=0 2024-12-05T11:57:04,671 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:57:04,674 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205cbedf535679c41c696014d47e4ab278e_cde5c157572c2beb0bd959b68c61d4ee store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:57:04,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742035_1211 (size=13017) 2024-12-05T11:57:04,681 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/19f8310c9eb04900931768533a06d091 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/19f8310c9eb04900931768533a06d091 2024-12-05T11:57:04,686 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/B of cde5c157572c2beb0bd959b68c61d4ee into 19f8310c9eb04900931768533a06d091(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:04,686 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:57:04,686 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/B, priority=12, startTime=1733399824658; duration=0sec 2024-12-05T11:57:04,686 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:04,686 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:B 2024-12-05T11:57:04,686 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:57:04,687 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:57:04,688 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): cde5c157572c2beb0bd959b68c61d4ee/C is initiating minor compaction (all files) 2024-12-05T11:57:04,688 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cde5c157572c2beb0bd959b68c61d4ee/C in TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,688 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c008762ef7c749ccad2c7b3e581dcc42, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/ab90c7418ef34889929f69f806157e8f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a5f5d0fe89654173a871fc80018a573c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/373f64e903df41d6902c4b7da0ee09cb] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp, totalSize=48.5 K 2024-12-05T11:57:04,688 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c008762ef7c749ccad2c7b3e581dcc42, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733399817830 2024-12-05T11:57:04,688 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ab90c7418ef34889929f69f806157e8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733399818999 2024-12-05T11:57:04,689 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting a5f5d0fe89654173a871fc80018a573c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733399819643 2024-12-05T11:57:04,689 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 373f64e903df41d6902c4b7da0ee09cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733399821837 2024-12-05T11:57:04,699 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205cbedf535679c41c696014d47e4ab278e_cde5c157572c2beb0bd959b68c61d4ee, store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:57:04,699 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cbedf535679c41c696014d47e4ab278e_cde5c157572c2beb0bd959b68c61d4ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:57:04,700 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#C#compaction#176 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:04,700 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/56e8bd59e36e41588ec4fe039ba941d4 is 50, key is test_row_0/C:col10/1733399821837/Put/seqid=0 2024-12-05T11:57:04,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742036_1212 (size=4469) 2024-12-05T11:57:04,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742037_1213 (size=13017) 2024-12-05T11:57:04,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T11:57:04,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:04,726 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T11:57:04,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a2f8bbfd20494239a5c54e254669a2af_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399822998/Put/seqid=0 2024-12-05T11:57:04,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742038_1214 (size=12454) 2024-12-05T11:57:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T11:57:05,106 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cde5c157572c2beb0bd959b68c61d4ee#A#compaction#175 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:05,108 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/662ebf1142c74a4aabc2920c0faa0c22 is 175, key is test_row_0/A:col10/1733399821837/Put/seqid=0 2024-12-05T11:57:05,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742039_1215 (size=31971) 2024-12-05T11:57:05,116 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/56e8bd59e36e41588ec4fe039ba941d4 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/56e8bd59e36e41588ec4fe039ba941d4 2024-12-05T11:57:05,121 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/C of cde5c157572c2beb0bd959b68c61d4ee into 56e8bd59e36e41588ec4fe039ba941d4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:05,121 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:57:05,121 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/C, priority=12, startTime=1733399824658; duration=0sec 2024-12-05T11:57:05,121 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:05,121 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:C 2024-12-05T11:57:05,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:05,145 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a2f8bbfd20494239a5c54e254669a2af_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a2f8bbfd20494239a5c54e254669a2af_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:05,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/da57fd4786294ab99fdfb528cf9df41f, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:57:05,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/da57fd4786294ab99fdfb528cf9df41f is 175, key is test_row_0/A:col10/1733399822998/Put/seqid=0 2024-12-05T11:57:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:05,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. as already flushing 2024-12-05T11:57:05,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742040_1216 (size=31255) 2024-12-05T11:57:05,150 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-12-05T11:57:05,150 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:05,150 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:05,150 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:05,150 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:05,159 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-12-05T11:57:05,159 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:05,159 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:05,159 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:05,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:05,525 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/662ebf1142c74a4aabc2920c0faa0c22 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/662ebf1142c74a4aabc2920c0faa0c22 2024-12-05T11:57:05,530 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cde5c157572c2beb0bd959b68c61d4ee/A of cde5c157572c2beb0bd959b68c61d4ee into 662ebf1142c74a4aabc2920c0faa0c22(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:05,530 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:57:05,531 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., storeName=cde5c157572c2beb0bd959b68c61d4ee/A, priority=12, startTime=1733399824658; duration=0sec 2024-12-05T11:57:05,531 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:05,531 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cde5c157572c2beb0bd959b68c61d4ee:A 2024-12-05T11:57:05,550 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/da57fd4786294ab99fdfb528cf9df41f 2024-12-05T11:57:05,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/bd3b5ca76bc5429b86ec851cbe0c068a is 50, key is test_row_0/B:col10/1733399822998/Put/seqid=0 2024-12-05T11:57:05,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742041_1217 (size=12301) 2024-12-05T11:57:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T11:57:05,964 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/bd3b5ca76bc5429b86ec851cbe0c068a 2024-12-05T11:57:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/f5e58959d9e241edbf2785321708285b is 50, key is test_row_0/C:col10/1733399822998/Put/seqid=0 2024-12-05T11:57:05,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742042_1218 (size=12301) 2024-12-05T11:57:06,375 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/f5e58959d9e241edbf2785321708285b 2024-12-05T11:57:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/da57fd4786294ab99fdfb528cf9df41f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/da57fd4786294ab99fdfb528cf9df41f 2024-12-05T11:57:06,390 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/da57fd4786294ab99fdfb528cf9df41f, entries=150, sequenceid=332, filesize=30.5 K 2024-12-05T11:57:06,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/bd3b5ca76bc5429b86ec851cbe0c068a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bd3b5ca76bc5429b86ec851cbe0c068a 2024-12-05T11:57:06,396 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bd3b5ca76bc5429b86ec851cbe0c068a, entries=150, sequenceid=332, filesize=12.0 K 2024-12-05T11:57:06,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/f5e58959d9e241edbf2785321708285b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/f5e58959d9e241edbf2785321708285b 2024-12-05T11:57:06,402 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/f5e58959d9e241edbf2785321708285b, entries=150, sequenceid=332, filesize=12.0 K 2024-12-05T11:57:06,403 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=13.42 KB/13740 for cde5c157572c2beb0bd959b68c61d4ee in 1677ms, sequenceid=332, compaction requested=false 2024-12-05T11:57:06,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for cde5c157572c2beb0bd959b68c61d4ee: 2024-12-05T11:57:06,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:06,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-05T11:57:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-05T11:57:06,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-05T11:57:06,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6130 sec 2024-12-05T11:57:06,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.6180 sec 2024-12-05T11:57:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T11:57:07,968 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:08,034 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T11:57:14,059 INFO [Thread-640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-640. 2024-12-05T11:57:14,059 DEBUG [Thread-640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:14,059 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:14,059 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:14,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:14,131 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-12-05T11:57:14,132 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:14,132 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:14,132 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:14,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:14,149 INFO [Thread-632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-632. 2024-12-05T11:57:14,149 DEBUG [Thread-632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:14,149 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:14,149 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:14,149 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:14,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T11:57:14,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-12-05T11:57:14,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-12-05T11:57:14,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-05T11:57:14,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-12-05T11:57:14,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-12-05T11:57:14,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T11:57:14,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5028 2024-12-05T11:57:14,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4978 2024-12-05T11:57:14,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T11:57:14,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2229 2024-12-05T11:57:14,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6685 rows 2024-12-05T11:57:14,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2223 2024-12-05T11:57:14,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6664 rows 2024-12-05T11:57:14,150 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:57:14,150 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:57:14,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:14,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:14,150 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:14,150 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T11:57:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T11:57:14,153 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399834153"}]},"ts":"1733399834153"} 2024-12-05T11:57:14,154 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T11:57:14,154 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T11:57:14,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:57:14,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, UNASSIGN}] 2024-12-05T11:57:14,156 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, UNASSIGN 2024-12-05T11:57:14,157 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=cde5c157572c2beb0bd959b68c61d4ee, regionState=CLOSING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:57:14,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, UNASSIGN because future has completed 2024-12-05T11:57:14,158 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:57:14,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:57:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T11:57:14,311 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:14,311 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:57:14,311 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing cde5c157572c2beb0bd959b68c61d4ee, disabling compactions & flushes 2024-12-05T11:57:14,311 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:14,311 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:14,311 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. after waiting 0 ms 2024-12-05T11:57:14,311 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:14,312 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing cde5c157572c2beb0bd959b68c61d4ee 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-05T11:57:14,321 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052fbd9f41b9d041bb992b3d49b7a639c1_cde5c157572c2beb0bd959b68c61d4ee is 50, key is test_row_0/A:col10/1733399834129/Put/seqid=0 2024-12-05T11:57:14,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742043_1219 (size=12454) 2024-12-05T11:57:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T11:57:14,727 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:14,739 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052fbd9f41b9d041bb992b3d49b7a639c1_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052fbd9f41b9d041bb992b3d49b7a639c1_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:14,741 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/0e3192dc71a04323a7486e42896b6276, store: [table=TestAcidGuarantees family=A region=cde5c157572c2beb0bd959b68c61d4ee] 2024-12-05T11:57:14,741 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/0e3192dc71a04323a7486e42896b6276 is 175, key is test_row_0/A:col10/1733399834129/Put/seqid=0 2024-12-05T11:57:14,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742044_1220 (size=31255) 2024-12-05T11:57:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T11:57:15,147 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/0e3192dc71a04323a7486e42896b6276 2024-12-05T11:57:15,161 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/2b01e469860f40adb7630985cfb26bdb is 50, key is test_row_0/B:col10/1733399834129/Put/seqid=0 2024-12-05T11:57:15,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742045_1221 (size=12301) 2024-12-05T11:57:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T11:57:15,567 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/2b01e469860f40adb7630985cfb26bdb 2024-12-05T11:57:15,583 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/e29d16feda98445f9dd950e6e75e36b1 is 50, key is test_row_0/C:col10/1733399834129/Put/seqid=0 2024-12-05T11:57:15,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742046_1222 (size=12301) 2024-12-05T11:57:15,987 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/e29d16feda98445f9dd950e6e75e36b1 2024-12-05T11:57:15,993 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/A/0e3192dc71a04323a7486e42896b6276 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0e3192dc71a04323a7486e42896b6276 2024-12-05T11:57:15,997 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0e3192dc71a04323a7486e42896b6276, entries=150, sequenceid=342, filesize=30.5 K 2024-12-05T11:57:15,998 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/B/2b01e469860f40adb7630985cfb26bdb as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/2b01e469860f40adb7630985cfb26bdb 2024-12-05T11:57:16,001 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/2b01e469860f40adb7630985cfb26bdb, entries=150, sequenceid=342, filesize=12.0 K 2024-12-05T11:57:16,002 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/.tmp/C/e29d16feda98445f9dd950e6e75e36b1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/e29d16feda98445f9dd950e6e75e36b1 2024-12-05T11:57:16,006 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/e29d16feda98445f9dd950e6e75e36b1, entries=150, sequenceid=342, filesize=12.0 K 2024-12-05T11:57:16,007 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for cde5c157572c2beb0bd959b68c61d4ee in 1696ms, sequenceid=342, compaction requested=true 2024-12-05T11:57:16,007 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/20afad4dc7734efeb91fd50257838f2e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/1078b9cf6e4e4239be9c92b4fe1d4b76, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/dea7b107d5444ed0bbcb7ee15502e1fb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6a7e372ec89846639f19a7b1704334a1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/bb185a7051cb42428ae74962e7b99ad1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/46b598e910f1442b874912f2ee77fa10, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/a091559db89646179efca5dd6e554a3d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/90e6e771a1a2463294f1cef086eea194, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2ec971f80bf94ff1ae7d716f496b2574, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2d447913cfe04cb5ba6d44c729334447, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2171ede5a71049118a4d2ed5ff5275d2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0fb18a9ad1454385a2f7e684eb535aca, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/51f646340b42491e9ad14f9e0f7bd094, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/11874f1a44fe4f658a72bc8923c9c767, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2625d69cf0374699a88123a2732c9885, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2014b9d1ee31458bb25bf95569553b35, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/65dc4bb3c2fd48f7b30a657af92bb3d2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c41df24679984ec3b424ffde4a1cd21d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0e371a022fa548939c1d5eb8278e0903, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c8ad4a9a170145d091581e701bb9721f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6d3824ad2916464b936328837a443a2f] to archive 2024-12-05T11:57:16,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:57:16,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/20afad4dc7734efeb91fd50257838f2e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/20afad4dc7734efeb91fd50257838f2e 2024-12-05T11:57:16,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/1078b9cf6e4e4239be9c92b4fe1d4b76 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/1078b9cf6e4e4239be9c92b4fe1d4b76 2024-12-05T11:57:16,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/dea7b107d5444ed0bbcb7ee15502e1fb to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/dea7b107d5444ed0bbcb7ee15502e1fb 2024-12-05T11:57:16,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6a7e372ec89846639f19a7b1704334a1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6a7e372ec89846639f19a7b1704334a1 2024-12-05T11:57:16,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/bb185a7051cb42428ae74962e7b99ad1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/bb185a7051cb42428ae74962e7b99ad1 2024-12-05T11:57:16,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/46b598e910f1442b874912f2ee77fa10 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/46b598e910f1442b874912f2ee77fa10 2024-12-05T11:57:16,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/a091559db89646179efca5dd6e554a3d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/a091559db89646179efca5dd6e554a3d 2024-12-05T11:57:16,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/90e6e771a1a2463294f1cef086eea194 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/90e6e771a1a2463294f1cef086eea194 2024-12-05T11:57:16,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2ec971f80bf94ff1ae7d716f496b2574 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2ec971f80bf94ff1ae7d716f496b2574 2024-12-05T11:57:16,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2d447913cfe04cb5ba6d44c729334447 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2d447913cfe04cb5ba6d44c729334447 2024-12-05T11:57:16,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2171ede5a71049118a4d2ed5ff5275d2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2171ede5a71049118a4d2ed5ff5275d2 2024-12-05T11:57:16,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0fb18a9ad1454385a2f7e684eb535aca to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0fb18a9ad1454385a2f7e684eb535aca 2024-12-05T11:57:16,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/51f646340b42491e9ad14f9e0f7bd094 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/51f646340b42491e9ad14f9e0f7bd094 2024-12-05T11:57:16,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/11874f1a44fe4f658a72bc8923c9c767 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/11874f1a44fe4f658a72bc8923c9c767 2024-12-05T11:57:16,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2625d69cf0374699a88123a2732c9885 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2625d69cf0374699a88123a2732c9885 2024-12-05T11:57:16,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2014b9d1ee31458bb25bf95569553b35 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/2014b9d1ee31458bb25bf95569553b35 2024-12-05T11:57:16,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/65dc4bb3c2fd48f7b30a657af92bb3d2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/65dc4bb3c2fd48f7b30a657af92bb3d2 2024-12-05T11:57:16,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c41df24679984ec3b424ffde4a1cd21d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c41df24679984ec3b424ffde4a1cd21d 2024-12-05T11:57:16,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0e371a022fa548939c1d5eb8278e0903 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0e371a022fa548939c1d5eb8278e0903 2024-12-05T11:57:16,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c8ad4a9a170145d091581e701bb9721f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/c8ad4a9a170145d091581e701bb9721f 2024-12-05T11:57:16,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6d3824ad2916464b936328837a443a2f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/6d3824ad2916464b936328837a443a2f 2024-12-05T11:57:16,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0fd5f3ec9d004798959b77b671833e21, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/41b1a6858fd241298f2ee186037ed9dd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/a40ac5c6444c4d3bb0bb47886029b5b7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/2d5edb4777a94743bd44c958b414ba86, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/125026c1b21b43fc86ad5e339877c220, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/e173cbf94fcf415ea04eab72806ae1e6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/615aab50d7bc4036ba62277df8f1608e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/1b1db4a438bb44588d769879d9edff9d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0a3dc046df944145a9c452303fa21108, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bb0c0f2d719c4090bbd5c0d287fb0edb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/c0d45fcba7f84f0cacc235c13f7285d0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0288a714be904057a2d171ff2f7531fd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bac44bcbf0584eb28e8c51a188ba920a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/4730085af93947be89fde2f2cbfba978, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0dbf79e815e843d88230118e7fce60ac, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/81c73b8ab48740fd9f82c213ce556c70, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/03d703be4cb846589d6d72d72d2a5482, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/b645ab0a06454444b72fc9bf21823858, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/8d676937f27049049a885b47fa831e78, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/502c5c3e408242828dc1eebb029cbb17, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/5eab5d3353ef42d69796fc2e0275ed2a] to archive 2024-12-05T11:57:16,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:57:16,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0fd5f3ec9d004798959b77b671833e21 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0fd5f3ec9d004798959b77b671833e21 2024-12-05T11:57:16,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/41b1a6858fd241298f2ee186037ed9dd to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/41b1a6858fd241298f2ee186037ed9dd 2024-12-05T11:57:16,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/a40ac5c6444c4d3bb0bb47886029b5b7 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/a40ac5c6444c4d3bb0bb47886029b5b7 2024-12-05T11:57:16,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/2d5edb4777a94743bd44c958b414ba86 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/2d5edb4777a94743bd44c958b414ba86 2024-12-05T11:57:16,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/125026c1b21b43fc86ad5e339877c220 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/125026c1b21b43fc86ad5e339877c220 2024-12-05T11:57:16,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/e173cbf94fcf415ea04eab72806ae1e6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/e173cbf94fcf415ea04eab72806ae1e6 2024-12-05T11:57:16,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/615aab50d7bc4036ba62277df8f1608e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/615aab50d7bc4036ba62277df8f1608e 2024-12-05T11:57:16,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/1b1db4a438bb44588d769879d9edff9d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/1b1db4a438bb44588d769879d9edff9d 2024-12-05T11:57:16,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0a3dc046df944145a9c452303fa21108 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0a3dc046df944145a9c452303fa21108 2024-12-05T11:57:16,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bb0c0f2d719c4090bbd5c0d287fb0edb to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bb0c0f2d719c4090bbd5c0d287fb0edb 2024-12-05T11:57:16,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/c0d45fcba7f84f0cacc235c13f7285d0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/c0d45fcba7f84f0cacc235c13f7285d0 2024-12-05T11:57:16,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0288a714be904057a2d171ff2f7531fd to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0288a714be904057a2d171ff2f7531fd 2024-12-05T11:57:16,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bac44bcbf0584eb28e8c51a188ba920a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bac44bcbf0584eb28e8c51a188ba920a 2024-12-05T11:57:16,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/4730085af93947be89fde2f2cbfba978 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/4730085af93947be89fde2f2cbfba978 2024-12-05T11:57:16,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0dbf79e815e843d88230118e7fce60ac to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/0dbf79e815e843d88230118e7fce60ac 2024-12-05T11:57:16,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/81c73b8ab48740fd9f82c213ce556c70 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/81c73b8ab48740fd9f82c213ce556c70 2024-12-05T11:57:16,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/03d703be4cb846589d6d72d72d2a5482 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/03d703be4cb846589d6d72d72d2a5482 2024-12-05T11:57:16,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/b645ab0a06454444b72fc9bf21823858 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/b645ab0a06454444b72fc9bf21823858 2024-12-05T11:57:16,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/8d676937f27049049a885b47fa831e78 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/8d676937f27049049a885b47fa831e78 2024-12-05T11:57:16,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/502c5c3e408242828dc1eebb029cbb17 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/502c5c3e408242828dc1eebb029cbb17 2024-12-05T11:57:16,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/5eab5d3353ef42d69796fc2e0275ed2a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/5eab5d3353ef42d69796fc2e0275ed2a 2024-12-05T11:57:16,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/5ed68ff89c2e4c20811aa98f8779e10a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c463e01d23484ec098499b01c66d718b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c0f4669b1e154c2f8a20fc3244b6bc0e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a6ab94c1ab5649bfa3b104d7cb9de7e1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/7de4d61bbcff4b139e7cc31ef61977ad, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/5b937007797e4876ad784fc380822bd4, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/f3403f0943e84c3ea88c8083ea3cbf81, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/792a8a04a3a34e21922e384aaa45405b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/fa6bd63cfffc493992b11638a82b6f5a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a559022980ea4efda6ac975a73c856a2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/38ebf032d2a74783afb132ea15ccfc30, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/7d6567bbc13543e29de90912a5b0d7c1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/b54ffb0492434be9a7072bcdb7981e16, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/eb7d45b4722a468e93586cc0d3b48001, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/e9cb2ae6631e41489878a3aaf5dbceee, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/486b9e68caef437e8986e50ef2c3f3fa, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c008762ef7c749ccad2c7b3e581dcc42, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/8a69b67220a94e2d8bf8c70537e6139c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/ab90c7418ef34889929f69f806157e8f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a5f5d0fe89654173a871fc80018a573c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/373f64e903df41d6902c4b7da0ee09cb] to archive 2024-12-05T11:57:16,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:57:16,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/5ed68ff89c2e4c20811aa98f8779e10a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/5ed68ff89c2e4c20811aa98f8779e10a 2024-12-05T11:57:16,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c463e01d23484ec098499b01c66d718b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c463e01d23484ec098499b01c66d718b 2024-12-05T11:57:16,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c0f4669b1e154c2f8a20fc3244b6bc0e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c0f4669b1e154c2f8a20fc3244b6bc0e 2024-12-05T11:57:16,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a6ab94c1ab5649bfa3b104d7cb9de7e1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a6ab94c1ab5649bfa3b104d7cb9de7e1 2024-12-05T11:57:16,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/7de4d61bbcff4b139e7cc31ef61977ad to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/7de4d61bbcff4b139e7cc31ef61977ad 2024-12-05T11:57:16,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/5b937007797e4876ad784fc380822bd4 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/5b937007797e4876ad784fc380822bd4 2024-12-05T11:57:16,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/f3403f0943e84c3ea88c8083ea3cbf81 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/f3403f0943e84c3ea88c8083ea3cbf81 2024-12-05T11:57:16,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/792a8a04a3a34e21922e384aaa45405b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/792a8a04a3a34e21922e384aaa45405b 2024-12-05T11:57:16,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/fa6bd63cfffc493992b11638a82b6f5a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/fa6bd63cfffc493992b11638a82b6f5a 2024-12-05T11:57:16,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a559022980ea4efda6ac975a73c856a2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a559022980ea4efda6ac975a73c856a2 2024-12-05T11:57:16,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/38ebf032d2a74783afb132ea15ccfc30 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/38ebf032d2a74783afb132ea15ccfc30 2024-12-05T11:57:16,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/7d6567bbc13543e29de90912a5b0d7c1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/7d6567bbc13543e29de90912a5b0d7c1 2024-12-05T11:57:16,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/b54ffb0492434be9a7072bcdb7981e16 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/b54ffb0492434be9a7072bcdb7981e16 2024-12-05T11:57:16,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/eb7d45b4722a468e93586cc0d3b48001 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/eb7d45b4722a468e93586cc0d3b48001 2024-12-05T11:57:16,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/e9cb2ae6631e41489878a3aaf5dbceee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/e9cb2ae6631e41489878a3aaf5dbceee 2024-12-05T11:57:16,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/486b9e68caef437e8986e50ef2c3f3fa to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/486b9e68caef437e8986e50ef2c3f3fa 2024-12-05T11:57:16,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c008762ef7c749ccad2c7b3e581dcc42 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/c008762ef7c749ccad2c7b3e581dcc42 2024-12-05T11:57:16,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/8a69b67220a94e2d8bf8c70537e6139c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/8a69b67220a94e2d8bf8c70537e6139c 2024-12-05T11:57:16,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/ab90c7418ef34889929f69f806157e8f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/ab90c7418ef34889929f69f806157e8f 2024-12-05T11:57:16,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a5f5d0fe89654173a871fc80018a573c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/a5f5d0fe89654173a871fc80018a573c 2024-12-05T11:57:16,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/373f64e903df41d6902c4b7da0ee09cb to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/373f64e903df41d6902c4b7da0ee09cb 2024-12-05T11:57:16,075 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/recovered.edits/345.seqid, newMaxSeqId=345, maxSeqId=4 2024-12-05T11:57:16,075 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. 2024-12-05T11:57:16,075 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for cde5c157572c2beb0bd959b68c61d4ee: Waiting for close lock at 1733399834311Running coprocessor pre-close hooks at 1733399834311Disabling compacts and flushes for region at 1733399834311Disabling writes for close at 1733399834311Obtaining lock to block concurrent updates at 1733399834312 (+1 ms)Preparing flush snapshotting stores in cde5c157572c2beb0bd959b68c61d4ee at 1733399834312Finished memstore snapshotting TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733399834312Flushing stores of TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee. at 1733399834313 (+1 ms)Flushing cde5c157572c2beb0bd959b68c61d4ee/A: creating writer at 1733399834313Flushing mob file cde5c157572c2beb0bd959b68c61d4ee/A: appending metadata at 1733399834321 (+8 ms)Flushing mob file cde5c157572c2beb0bd959b68c61d4ee/A: closing flushed file at 1733399834321Flushing cde5c157572c2beb0bd959b68c61d4ee/A: appending metadata at 1733399834741 (+420 ms)Flushing cde5c157572c2beb0bd959b68c61d4ee/A: closing flushed file at 1733399834741Flushing cde5c157572c2beb0bd959b68c61d4ee/B: creating writer at 1733399835156 (+415 ms)Flushing cde5c157572c2beb0bd959b68c61d4ee/B: appending metadata at 1733399835160 (+4 ms)Flushing cde5c157572c2beb0bd959b68c61d4ee/B: closing flushed file at 1733399835160Flushing cde5c157572c2beb0bd959b68c61d4ee/C: creating writer at 1733399835578 (+418 ms)Flushing cde5c157572c2beb0bd959b68c61d4ee/C: appending metadata at 1733399835582 (+4 ms)Flushing cde5c157572c2beb0bd959b68c61d4ee/C: closing flushed file at 1733399835582Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1912c8d9: reopening flushed file at 1733399835992 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e1a500d: reopening flushed file at 1733399835997 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37b7dbff: reopening flushed file at 1733399836001 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for cde5c157572c2beb0bd959b68c61d4ee in 1696ms, sequenceid=342, compaction requested=true at 1733399836007 (+6 ms)Writing region close event to WAL at 1733399836072 (+65 ms)Running coprocessor post-close hooks at 1733399836075 (+3 ms)Closed at 1733399836075 2024-12-05T11:57:16,077 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,078 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=cde5c157572c2beb0bd959b68c61d4ee, regionState=CLOSED 2024-12-05T11:57:16,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:57:16,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-05T11:57:16,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure cde5c157572c2beb0bd959b68c61d4ee, server=80666a11a09f,40121,1733399769572 in 1.9210 sec 2024-12-05T11:57:16,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-05T11:57:16,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cde5c157572c2beb0bd959b68c61d4ee, UNASSIGN in 1.9260 sec 2024-12-05T11:57:16,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-05T11:57:16,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9290 sec 2024-12-05T11:57:16,085 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399836085"}]},"ts":"1733399836085"} 2024-12-05T11:57:16,086 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T11:57:16,086 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T11:57:16,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9370 sec 2024-12-05T11:57:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T11:57:16,298 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:16,299 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T11:57:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:16,300 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T11:57:16,301 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:16,303 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,306 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/recovered.edits] 2024-12-05T11:57:16,308 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0e3192dc71a04323a7486e42896b6276 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/0e3192dc71a04323a7486e42896b6276 2024-12-05T11:57:16,309 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/662ebf1142c74a4aabc2920c0faa0c22 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/662ebf1142c74a4aabc2920c0faa0c22 2024-12-05T11:57:16,310 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/da57fd4786294ab99fdfb528cf9df41f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/A/da57fd4786294ab99fdfb528cf9df41f 2024-12-05T11:57:16,312 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/19f8310c9eb04900931768533a06d091 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/19f8310c9eb04900931768533a06d091 2024-12-05T11:57:16,313 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/2b01e469860f40adb7630985cfb26bdb to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/2b01e469860f40adb7630985cfb26bdb 2024-12-05T11:57:16,315 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bd3b5ca76bc5429b86ec851cbe0c068a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/B/bd3b5ca76bc5429b86ec851cbe0c068a 2024-12-05T11:57:16,318 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/56e8bd59e36e41588ec4fe039ba941d4 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/56e8bd59e36e41588ec4fe039ba941d4 2024-12-05T11:57:16,319 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/e29d16feda98445f9dd950e6e75e36b1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/e29d16feda98445f9dd950e6e75e36b1 2024-12-05T11:57:16,320 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/f5e58959d9e241edbf2785321708285b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/C/f5e58959d9e241edbf2785321708285b 2024-12-05T11:57:16,324 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/recovered.edits/345.seqid to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee/recovered.edits/345.seqid 2024-12-05T11:57:16,324 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,325 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T11:57:16,325 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T11:57:16,326 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T11:57:16,332 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205036c26a0b0d4471e98a95f23b1caea51_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205036c26a0b0d4471e98a95f23b1caea51_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,334 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050df0572151504a60880e7df7a8dbaf01_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050df0572151504a60880e7df7a8dbaf01_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,335 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052fbd9f41b9d041bb992b3d49b7a639c1_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052fbd9f41b9d041bb992b3d49b7a639c1_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,336 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205374f251f387b468fbd792744c8f27d2a_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205374f251f387b468fbd792744c8f27d2a_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,338 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205414facfcf42d478080867bde102d17b4_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205414facfcf42d478080867bde102d17b4_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,339 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120542ad50be917e478abd304832c58694cf_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120542ad50be917e478abd304832c58694cf_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,341 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055eeac40d22fd4d0da2d7448c4707b42d_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055eeac40d22fd4d0da2d7448c4707b42d_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,343 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057ce52aa74ca14ff5acbd27a12fb66caf_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057ce52aa74ca14ff5acbd27a12fb66caf_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,347 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205804d6bbb7e614e0090d7b4a7830800b4_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205804d6bbb7e614e0090d7b4a7830800b4_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,350 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205855e6c32737d488891e8f36130af2737_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205855e6c32737d488891e8f36130af2737_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,352 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058a50933ccc1f4cb5bc9856dba4209eea_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058a50933ccc1f4cb5bc9856dba4209eea_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,356 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058e8e6dfc5eb24649be9476ae4ef41196_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058e8e6dfc5eb24649be9476ae4ef41196_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,357 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205952cd0bf0e4c466f80d3a94f055c724e_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205952cd0bf0e4c466f80d3a94f055c724e_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,359 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a2f8bbfd20494239a5c54e254669a2af_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a2f8bbfd20494239a5c54e254669a2af_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,360 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ada3d6ce0efe480b8db58f873c5eb3f3_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ada3d6ce0efe480b8db58f873c5eb3f3_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,361 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bcae8ef5c8384c1bb2cabc985190edf7_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bcae8ef5c8384c1bb2cabc985190edf7_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,363 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205de2b7ff34b70465b9fcf09bcbf684e23_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205de2b7ff34b70465b9fcf09bcbf684e23_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,364 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ff52a6cad26b45729ad35eca13724848_cde5c157572c2beb0bd959b68c61d4ee to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ff52a6cad26b45729ad35eca13724848_cde5c157572c2beb0bd959b68c61d4ee 2024-12-05T11:57:16,374 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T11:57:16,381 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:16,384 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T11:57:16,387 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T11:57:16,389 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:16,389 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T11:57:16,389 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733399836389"}]},"ts":"9223372036854775807"} 2024-12-05T11:57:16,392 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T11:57:16,393 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cde5c157572c2beb0bd959b68c61d4ee, NAME => 'TestAcidGuarantees,,1733399801038.cde5c157572c2beb0bd959b68c61d4ee.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T11:57:16,393 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T11:57:16,393 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733399836393"}]},"ts":"9223372036854775807"} 2024-12-05T11:57:16,396 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T11:57:16,397 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:16,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 98 msec 2024-12-05T11:57:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T11:57:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T11:57:16,410 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:16,419 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 238), OpenFileDescriptor=445 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=272 (was 288), ProcessCount=11 (was 11), AvailableMemoryMB=6702 (was 6909) 2024-12-05T11:57:16,432 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=6702 2024-12-05T11:57:16,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:57:16,433 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:57:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:16,436 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T11:57:16,436 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:16,436 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-12-05T11:57:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-05T11:57:16,437 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T11:57:16,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742047_1223 (size=959) 2024-12-05T11:57:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-05T11:57:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-05T11:57:16,851 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c 2024-12-05T11:57:16,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742048_1224 (size=53) 2024-12-05T11:57:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-05T11:57:17,263 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:57:17,263 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d5e0da544abea90f443151cdd8418bf7, disabling compactions & flushes 2024-12-05T11:57:17,263 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:17,263 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:17,263 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. after waiting 0 ms 2024-12-05T11:57:17,263 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:17,263 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:17,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d5e0da544abea90f443151cdd8418bf7: Waiting for close lock at 1733399837263Disabling compacts and flushes for region at 1733399837263Disabling writes for close at 1733399837263Writing region close event to WAL at 1733399837263Closed at 1733399837263 2024-12-05T11:57:17,266 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T11:57:17,267 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733399837267"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733399837267"}]},"ts":"1733399837267"} 2024-12-05T11:57:17,271 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T11:57:17,273 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T11:57:17,274 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399837274"}]},"ts":"1733399837274"} 2024-12-05T11:57:17,278 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T11:57:17,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5e0da544abea90f443151cdd8418bf7, ASSIGN}] 2024-12-05T11:57:17,279 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5e0da544abea90f443151cdd8418bf7, ASSIGN 2024-12-05T11:57:17,280 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5e0da544abea90f443151cdd8418bf7, ASSIGN; state=OFFLINE, location=80666a11a09f,40121,1733399769572; forceNewPlan=false, retain=false 2024-12-05T11:57:17,431 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=d5e0da544abea90f443151cdd8418bf7, regionState=OPENING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:57:17,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5e0da544abea90f443151cdd8418bf7, ASSIGN because future has completed 2024-12-05T11:57:17,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:57:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-05T11:57:17,598 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:17,599 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:57:17,600 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,600 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:57:17,600 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,600 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,602 INFO [StoreOpener-d5e0da544abea90f443151cdd8418bf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,604 INFO [StoreOpener-d5e0da544abea90f443151cdd8418bf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5e0da544abea90f443151cdd8418bf7 columnFamilyName A 2024-12-05T11:57:17,604 DEBUG [StoreOpener-d5e0da544abea90f443151cdd8418bf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:17,605 INFO [StoreOpener-d5e0da544abea90f443151cdd8418bf7-1 {}] regionserver.HStore(327): Store=d5e0da544abea90f443151cdd8418bf7/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:57:17,605 INFO [StoreOpener-d5e0da544abea90f443151cdd8418bf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,606 INFO [StoreOpener-d5e0da544abea90f443151cdd8418bf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5e0da544abea90f443151cdd8418bf7 columnFamilyName B 2024-12-05T11:57:17,606 DEBUG [StoreOpener-d5e0da544abea90f443151cdd8418bf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:17,607 INFO [StoreOpener-d5e0da544abea90f443151cdd8418bf7-1 {}] regionserver.HStore(327): Store=d5e0da544abea90f443151cdd8418bf7/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:57:17,607 INFO [StoreOpener-d5e0da544abea90f443151cdd8418bf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,609 INFO [StoreOpener-d5e0da544abea90f443151cdd8418bf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5e0da544abea90f443151cdd8418bf7 columnFamilyName C 2024-12-05T11:57:17,609 DEBUG [StoreOpener-d5e0da544abea90f443151cdd8418bf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:17,609 INFO [StoreOpener-d5e0da544abea90f443151cdd8418bf7-1 {}] regionserver.HStore(327): Store=d5e0da544abea90f443151cdd8418bf7/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:57:17,609 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,610 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,611 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,612 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,612 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,613 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:57:17,615 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,618 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:57:17,619 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened d5e0da544abea90f443151cdd8418bf7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70184849, jitterRate=0.04583574831485748}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:57:17,619 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:17,619 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for d5e0da544abea90f443151cdd8418bf7: Running coprocessor pre-open hook at 1733399837601Writing region info on filesystem at 1733399837601Initializing all the Stores at 1733399837602 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399837602Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399837602Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399837602Cleaning up temporary data from old regions at 1733399837612 (+10 ms)Running coprocessor post-open hooks at 1733399837619 (+7 ms)Region opened successfully at 1733399837619 2024-12-05T11:57:17,620 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., pid=61, masterSystemTime=1733399837590 2024-12-05T11:57:17,623 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:17,623 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:17,623 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=d5e0da544abea90f443151cdd8418bf7, regionState=OPEN, openSeqNum=2, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:57:17,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:57:17,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-05T11:57:17,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 in 190 msec 2024-12-05T11:57:17,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-05T11:57:17,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5e0da544abea90f443151cdd8418bf7, ASSIGN in 350 msec 2024-12-05T11:57:17,632 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T11:57:17,632 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399837632"}]},"ts":"1733399837632"} 2024-12-05T11:57:17,634 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T11:57:17,635 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T11:57:17,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2010 sec 2024-12-05T11:57:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-05T11:57:18,589 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:18,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,590 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:18,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:18,592 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3793511e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:18,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:18,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:18,595 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb47cb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,597 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:18,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:18,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:18,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:18,600 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:57:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 80666a11a09f,33741,1733399768894 2024-12-05T11:57:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@265bdc56 2024-12-05T11:57:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:57:18,602 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:57:18,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:18,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:18,605 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d33ecee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:18,607 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b05f4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:18,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:18,609 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31aa940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:18,611 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d34c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:18,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:18,614 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:18,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:18,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:18,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33eae90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:18,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:18,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:18,616 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:18,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6305a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:18,618 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:18,619 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf11d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:18,621 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:18,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5730f705, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,622 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:18,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:18,623 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:18,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:18,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a5652d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:18,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:18,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:18,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b76d9e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:18,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:18,628 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103e98d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:18,630 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43a60d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,631 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:18,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:18,632 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@628e2aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:18,633 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37658fc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:18,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:18,635 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f65584, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:18,636 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53fa97c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:18,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:18,638 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6418ad69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:18,639 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68f9b778, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:18,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:18,640 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279bd8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:18,641 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ba749b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:18,642 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:18,642 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:18,642 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:18,642 DEBUG [Thread-1065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:18,642 DEBUG [Thread-1069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:18,642 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:18,643 DEBUG [Thread-1063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:18,643 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-05T11:57:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:18,644 DEBUG [Thread-1067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:18,644 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:18,645 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T11:57:18,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:18,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:18,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:18,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:18,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:18,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:18,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:18,647 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:18,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:18,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:18,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:18,648 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:57:18,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:57:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:57:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:57:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:57:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:57:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:57:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:57:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:57:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:57:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:18,657 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:18,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T11:57:18,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/f0fad698219b4b1e8e4b99a37be8362a is 50, key is test_row_0/A:col10/1733399838656/Put/seqid=0 2024-12-05T11:57:18,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399898681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399898681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:18,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399898682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:18,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399898683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:18,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399898684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:18,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742049_1225 (size=12001) 2024-12-05T11:57:18,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/f0fad698219b4b1e8e4b99a37be8362a 2024-12-05T11:57:18,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/a06dc8a7b4e447e68951a5851b42c295 is 50, key is test_row_0/B:col10/1733399838656/Put/seqid=0 2024-12-05T11:57:18,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742050_1226 (size=12001) 2024-12-05T11:57:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T11:57:18,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399898787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399898787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399898787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:18,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399898798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399898798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:18,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T11:57:18,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:18,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:18,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:18,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:18,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:18,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T11:57:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:18,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T11:57:18,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399898997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399898997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399898998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399899008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399899008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T11:57:19,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:19,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:19,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:19,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:19,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/a06dc8a7b4e447e68951a5851b42c295 2024-12-05T11:57:19,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/81f8a4fa3e7d4c3f9f7d0700d831261e is 50, key is test_row_0/C:col10/1733399838656/Put/seqid=0 2024-12-05T11:57:19,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742051_1227 (size=12001) 2024-12-05T11:57:19,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T11:57:19,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:19,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:19,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:19,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:19,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T11:57:19,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399899307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399899307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399899308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399899318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399899318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T11:57:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:19,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:19,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T11:57:19,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:19,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:19,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:19,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:19,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:19,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/81f8a4fa3e7d4c3f9f7d0700d831261e 2024-12-05T11:57:19,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/f0fad698219b4b1e8e4b99a37be8362a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/f0fad698219b4b1e8e4b99a37be8362a 2024-12-05T11:57:19,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/f0fad698219b4b1e8e4b99a37be8362a, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T11:57:19,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/a06dc8a7b4e447e68951a5851b42c295 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a06dc8a7b4e447e68951a5851b42c295 2024-12-05T11:57:19,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a06dc8a7b4e447e68951a5851b42c295, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T11:57:19,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/81f8a4fa3e7d4c3f9f7d0700d831261e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/81f8a4fa3e7d4c3f9f7d0700d831261e 2024-12-05T11:57:19,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/81f8a4fa3e7d4c3f9f7d0700d831261e, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T11:57:19,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for d5e0da544abea90f443151cdd8418bf7 in 964ms, sequenceid=14, compaction requested=false 2024-12-05T11:57:19,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:19,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T11:57:19,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:19,720 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T11:57:19,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/9097346942574861b1f0e19861e53c09 is 50, key is test_row_0/A:col10/1733399838681/Put/seqid=0 2024-12-05T11:57:19,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742052_1228 (size=12001) 2024-12-05T11:57:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T11:57:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:19,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399899826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399899828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399899828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399899828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399899829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399899938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399899938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399899938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,130 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/9097346942574861b1f0e19861e53c09 2024-12-05T11:57:20,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/b3e0b276933749b7ba232121787b8cb8 is 50, key is test_row_0/B:col10/1733399838681/Put/seqid=0 2024-12-05T11:57:20,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399900148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399900148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399900148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742053_1229 (size=12001) 2024-12-05T11:57:20,160 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/b3e0b276933749b7ba232121787b8cb8 2024-12-05T11:57:20,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ba61f1f28d144945bf176dbe848c030c is 50, key is test_row_0/C:col10/1733399838681/Put/seqid=0 2024-12-05T11:57:20,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742054_1230 (size=12001) 2024-12-05T11:57:20,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399900458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399900458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399900458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,570 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ba61f1f28d144945bf176dbe848c030c 2024-12-05T11:57:20,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/9097346942574861b1f0e19861e53c09 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/9097346942574861b1f0e19861e53c09 2024-12-05T11:57:20,578 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/9097346942574861b1f0e19861e53c09, entries=150, sequenceid=38, filesize=11.7 K 2024-12-05T11:57:20,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/b3e0b276933749b7ba232121787b8cb8 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b3e0b276933749b7ba232121787b8cb8 2024-12-05T11:57:20,582 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b3e0b276933749b7ba232121787b8cb8, entries=150, sequenceid=38, filesize=11.7 K 2024-12-05T11:57:20,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ba61f1f28d144945bf176dbe848c030c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ba61f1f28d144945bf176dbe848c030c 2024-12-05T11:57:20,587 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ba61f1f28d144945bf176dbe848c030c, entries=150, sequenceid=38, filesize=11.7 K 2024-12-05T11:57:20,588 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d5e0da544abea90f443151cdd8418bf7 in 868ms, sequenceid=38, compaction requested=false 2024-12-05T11:57:20,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:20,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:20,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-05T11:57:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-05T11:57:20,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-05T11:57:20,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9410 sec 2024-12-05T11:57:20,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.9490 sec 2024-12-05T11:57:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T11:57:20,788 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:20,788 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-05T11:57:20,790 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:57:20,791 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:20,791 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:20,840 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:20,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T11:57:20,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/2d738e02d58e4b6baf41a033e71f9f6d is 50, key is test_row_0/A:col10/1733399840839/Put/seqid=0 2024-12-05T11:57:20,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742055_1231 (size=16681) 2024-12-05T11:57:20,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399900872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399900875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:57:20,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:57:20,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:20,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:20,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:20,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:20,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:20,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399900967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399900967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399900967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399900978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:20,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399900988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:20,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:20,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:21,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:57:21,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:21,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:21,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:21,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:21,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:57:21,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399901188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:21,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:21,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399901198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:21,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:21,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:57:21,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:21,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:21,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/2d738e02d58e4b6baf41a033e71f9f6d 2024-12-05T11:57:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:21,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/27b67c0d32b2414db56e203de95fdad0 is 50, key is test_row_0/B:col10/1733399840839/Put/seqid=0 2024-12-05T11:57:21,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742056_1232 (size=12001) 2024-12-05T11:57:21,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:57:21,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:21,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:21,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:21,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:21,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:57:21,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399901498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:21,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399901507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:21,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:57:21,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:21,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:21,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:21,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:21,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:21,592 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:57:21,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/27b67c0d32b2414db56e203de95fdad0 2024-12-05T11:57:21,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ca4c021c78704b089146e246c2fa9b9e is 50, key is test_row_0/C:col10/1733399840839/Put/seqid=0 2024-12-05T11:57:21,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742057_1233 (size=12001) 2024-12-05T11:57:21,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ca4c021c78704b089146e246c2fa9b9e 2024-12-05T11:57:21,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/2d738e02d58e4b6baf41a033e71f9f6d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/2d738e02d58e4b6baf41a033e71f9f6d 2024-12-05T11:57:21,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/2d738e02d58e4b6baf41a033e71f9f6d, entries=250, sequenceid=52, filesize=16.3 K 2024-12-05T11:57:21,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/27b67c0d32b2414db56e203de95fdad0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/27b67c0d32b2414db56e203de95fdad0 2024-12-05T11:57:21,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/27b67c0d32b2414db56e203de95fdad0, entries=150, sequenceid=52, filesize=11.7 K 2024-12-05T11:57:21,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ca4c021c78704b089146e246c2fa9b9e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ca4c021c78704b089146e246c2fa9b9e 2024-12-05T11:57:21,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ca4c021c78704b089146e246c2fa9b9e, entries=150, sequenceid=52, filesize=11.7 K 2024-12-05T11:57:21,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d5e0da544abea90f443151cdd8418bf7 in 866ms, sequenceid=52, compaction requested=true 2024-12-05T11:57:21,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:21,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:57:21,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:21,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:57:21,706 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:21,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:21,706 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:21,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:57:21,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:21,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:57:21,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:21,707 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:21,707 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-05T11:57:21,707 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/B is initiating minor compaction (all files) 2024-12-05T11:57:21,707 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/B in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:21,707 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a06dc8a7b4e447e68951a5851b42c295, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b3e0b276933749b7ba232121787b8cb8, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/27b67c0d32b2414db56e203de95fdad0] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=35.2 K 2024-12-05T11:57:21,708 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting a06dc8a7b4e447e68951a5851b42c295, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733399838654 2024-12-05T11:57:21,708 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:21,708 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting b3e0b276933749b7ba232121787b8cb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733399838681 2024-12-05T11:57:21,708 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/A is initiating minor compaction (all files) 2024-12-05T11:57:21,709 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/A in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:21,709 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/f0fad698219b4b1e8e4b99a37be8362a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/9097346942574861b1f0e19861e53c09, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/2d738e02d58e4b6baf41a033e71f9f6d] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=39.7 K 2024-12-05T11:57:21,709 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 27b67c0d32b2414db56e203de95fdad0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399839825 2024-12-05T11:57:21,709 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0fad698219b4b1e8e4b99a37be8362a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733399838654 2024-12-05T11:57:21,709 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9097346942574861b1f0e19861e53c09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733399838681 2024-12-05T11:57:21,710 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d738e02d58e4b6baf41a033e71f9f6d, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399839822 2024-12-05T11:57:21,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/8b8aa3c7aa1443a19597963ca5d83072 is 50, key is test_row_0/A:col10/1733399840874/Put/seqid=0 2024-12-05T11:57:21,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742058_1234 (size=12001) 2024-12-05T11:57:21,717 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/8b8aa3c7aa1443a19597963ca5d83072 2024-12-05T11:57:21,723 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#A#compaction#193 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:21,724 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/8c6518978a464292b687c52290bf2161 is 50, key is test_row_0/A:col10/1733399840839/Put/seqid=0 2024-12-05T11:57:21,725 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#B#compaction#194 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:21,725 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/a62831be35cb4da996b3a2aa1c0a5796 is 50, key is test_row_0/B:col10/1733399840839/Put/seqid=0 2024-12-05T11:57:21,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/b2a70888a9a748dcab9e594a090ddcd6 is 50, key is test_row_0/B:col10/1733399840874/Put/seqid=0 2024-12-05T11:57:21,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742059_1235 (size=12104) 2024-12-05T11:57:21,747 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/8c6518978a464292b687c52290bf2161 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8c6518978a464292b687c52290bf2161 2024-12-05T11:57:21,753 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/A of d5e0da544abea90f443151cdd8418bf7 into 8c6518978a464292b687c52290bf2161(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:21,753 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:21,753 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/A, priority=13, startTime=1733399841706; duration=0sec 2024-12-05T11:57:21,754 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:21,754 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:A 2024-12-05T11:57:21,754 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:21,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742060_1236 (size=12104) 2024-12-05T11:57:21,756 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:21,756 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/C is initiating minor compaction (all files) 2024-12-05T11:57:21,756 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/C in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:21,756 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/81f8a4fa3e7d4c3f9f7d0700d831261e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ba61f1f28d144945bf176dbe848c030c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ca4c021c78704b089146e246c2fa9b9e] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=35.2 K 2024-12-05T11:57:21,757 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81f8a4fa3e7d4c3f9f7d0700d831261e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733399838654 2024-12-05T11:57:21,757 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba61f1f28d144945bf176dbe848c030c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733399838681 2024-12-05T11:57:21,757 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca4c021c78704b089146e246c2fa9b9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399839825 2024-12-05T11:57:21,761 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/a62831be35cb4da996b3a2aa1c0a5796 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a62831be35cb4da996b3a2aa1c0a5796 2024-12-05T11:57:21,767 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/B of d5e0da544abea90f443151cdd8418bf7 into a62831be35cb4da996b3a2aa1c0a5796(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:21,767 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:21,767 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/B, priority=13, startTime=1733399841706; duration=0sec 2024-12-05T11:57:21,767 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:21,767 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:B 2024-12-05T11:57:21,773 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#C#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:21,773 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/8625874616bb4c3798dd7cc8ef444646 is 50, key is test_row_0/C:col10/1733399840839/Put/seqid=0 2024-12-05T11:57:21,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742061_1237 (size=12001) 2024-12-05T11:57:21,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742062_1238 (size=12104) 2024-12-05T11:57:21,784 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/8625874616bb4c3798dd7cc8ef444646 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/8625874616bb4c3798dd7cc8ef444646 2024-12-05T11:57:21,791 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/C of d5e0da544abea90f443151cdd8418bf7 into 8625874616bb4c3798dd7cc8ef444646(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:21,791 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:21,791 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/C, priority=13, startTime=1733399841706; duration=0sec 2024-12-05T11:57:21,791 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:21,791 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:C 2024-12-05T11:57:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:57:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:21,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:22,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399901998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399902000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399902000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399902007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399902018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399902107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399902108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399902108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,176 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/b2a70888a9a748dcab9e594a090ddcd6 2024-12-05T11:57:22,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/a0d2d738064540569e16bd771d208cbe is 50, key is test_row_0/C:col10/1733399840874/Put/seqid=0 2024-12-05T11:57:22,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742063_1239 (size=12001) 2024-12-05T11:57:22,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399902317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399902317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399902317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,586 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/a0d2d738064540569e16bd771d208cbe 2024-12-05T11:57:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/8b8aa3c7aa1443a19597963ca5d83072 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8b8aa3c7aa1443a19597963ca5d83072 2024-12-05T11:57:22,595 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8b8aa3c7aa1443a19597963ca5d83072, entries=150, sequenceid=74, filesize=11.7 K 2024-12-05T11:57:22,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/b2a70888a9a748dcab9e594a090ddcd6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b2a70888a9a748dcab9e594a090ddcd6 2024-12-05T11:57:22,600 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b2a70888a9a748dcab9e594a090ddcd6, entries=150, sequenceid=74, filesize=11.7 K 2024-12-05T11:57:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/a0d2d738064540569e16bd771d208cbe as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/a0d2d738064540569e16bd771d208cbe 2024-12-05T11:57:22,604 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/a0d2d738064540569e16bd771d208cbe, entries=150, sequenceid=74, filesize=11.7 K 2024-12-05T11:57:22,605 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d5e0da544abea90f443151cdd8418bf7 in 897ms, sequenceid=74, compaction requested=false 2024-12-05T11:57:22,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:22,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:22,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-05T11:57:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-05T11:57:22,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-05T11:57:22,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-12-05T11:57:22,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.8200 sec 2024-12-05T11:57:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:22,629 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:22,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-05T11:57:22,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/3d11b265b6314bee86b233a1d6e26307 is 50, key is test_row_0/A:col10/1733399842629/Put/seqid=0 2024-12-05T11:57:22,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742064_1240 (size=14341) 2024-12-05T11:57:22,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399902648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399902650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399902651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399902758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399902758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399902758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:57:22,938 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:22,938 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-05T11:57:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T11:57:22,940 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:22,940 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:22,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:22,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399902968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399902968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:22,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399902968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:23,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399903018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:23,020 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in d5e0da544abea90f443151cdd8418bf7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:23,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399903028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:23,031 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in d5e0da544abea90f443151cdd8418bf7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:23,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/3d11b265b6314bee86b233a1d6e26307 2024-12-05T11:57:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T11:57:23,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/f6f8d2df02694b2db1c44d6f03cb1596 is 50, key is test_row_0/B:col10/1733399842629/Put/seqid=0 2024-12-05T11:57:23,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742065_1241 (size=12001) 2024-12-05T11:57:23,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T11:57:23,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:23,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T11:57:23,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:23,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T11:57:23,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399903278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:23,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399903278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399903278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:23,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T11:57:23,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:23,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/f6f8d2df02694b2db1c44d6f03cb1596 2024-12-05T11:57:23,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/24e0222dd7db42eb972811b658bf35b0 is 50, key is test_row_0/C:col10/1733399842629/Put/seqid=0 2024-12-05T11:57:23,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742066_1242 (size=12001) 2024-12-05T11:57:23,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T11:57:23,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:23,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T11:57:23,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T11:57:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399903787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:23,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399903787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399903788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:23,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T11:57:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:23,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/24e0222dd7db42eb972811b658bf35b0 2024-12-05T11:57:23,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/3d11b265b6314bee86b233a1d6e26307 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/3d11b265b6314bee86b233a1d6e26307 2024-12-05T11:57:23,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/3d11b265b6314bee86b233a1d6e26307, entries=200, sequenceid=94, filesize=14.0 K 2024-12-05T11:57:23,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/f6f8d2df02694b2db1c44d6f03cb1596 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f6f8d2df02694b2db1c44d6f03cb1596 2024-12-05T11:57:23,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f6f8d2df02694b2db1c44d6f03cb1596, entries=150, sequenceid=94, filesize=11.7 K 2024-12-05T11:57:23,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/24e0222dd7db42eb972811b658bf35b0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/24e0222dd7db42eb972811b658bf35b0 2024-12-05T11:57:23,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/24e0222dd7db42eb972811b658bf35b0, entries=150, sequenceid=94, filesize=11.7 K 2024-12-05T11:57:23,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for d5e0da544abea90f443151cdd8418bf7 in 1252ms, sequenceid=94, compaction requested=true 2024-12-05T11:57:23,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:23,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:57:23,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:23,882 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:23,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:57:23,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:23,882 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:23,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:57:23,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:23,883 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:23,883 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:23,883 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/B is initiating minor compaction (all files) 2024-12-05T11:57:23,883 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/A is initiating minor compaction (all files) 2024-12-05T11:57:23,883 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/A in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,883 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/B in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,883 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a62831be35cb4da996b3a2aa1c0a5796, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b2a70888a9a748dcab9e594a090ddcd6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f6f8d2df02694b2db1c44d6f03cb1596] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=35.3 K 2024-12-05T11:57:23,883 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8c6518978a464292b687c52290bf2161, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8b8aa3c7aa1443a19597963ca5d83072, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/3d11b265b6314bee86b233a1d6e26307] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=37.5 K 2024-12-05T11:57:23,883 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting a62831be35cb4da996b3a2aa1c0a5796, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399839825 2024-12-05T11:57:23,883 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c6518978a464292b687c52290bf2161, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399839825 2024-12-05T11:57:23,884 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting b2a70888a9a748dcab9e594a090ddcd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733399840857 2024-12-05T11:57:23,884 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b8aa3c7aa1443a19597963ca5d83072, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733399840857 2024-12-05T11:57:23,884 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting f6f8d2df02694b2db1c44d6f03cb1596, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399841995 2024-12-05T11:57:23,884 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d11b265b6314bee86b233a1d6e26307, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399841995 2024-12-05T11:57:23,891 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#A#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:23,892 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/1e9ad491c0b343a8a03715e3d67c1238 is 50, key is test_row_0/A:col10/1733399842629/Put/seqid=0 2024-12-05T11:57:23,896 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#B#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:23,896 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/11c714302fd0490e881bfaa70d0535c6 is 50, key is test_row_0/B:col10/1733399842629/Put/seqid=0 2024-12-05T11:57:23,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742067_1243 (size=12207) 2024-12-05T11:57:23,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742068_1244 (size=12207) 2024-12-05T11:57:23,937 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/11c714302fd0490e881bfaa70d0535c6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/11c714302fd0490e881bfaa70d0535c6 2024-12-05T11:57:23,943 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/B of d5e0da544abea90f443151cdd8418bf7 into 11c714302fd0490e881bfaa70d0535c6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:23,943 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:23,943 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/B, priority=13, startTime=1733399843882; duration=0sec 2024-12-05T11:57:23,943 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:23,943 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:B 2024-12-05T11:57:23,943 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:23,944 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:23,944 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/C is initiating minor compaction (all files) 2024-12-05T11:57:23,944 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/C in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:23,944 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/8625874616bb4c3798dd7cc8ef444646, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/a0d2d738064540569e16bd771d208cbe, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/24e0222dd7db42eb972811b658bf35b0] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=35.3 K 2024-12-05T11:57:23,945 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 8625874616bb4c3798dd7cc8ef444646, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733399839825 2024-12-05T11:57:23,945 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting a0d2d738064540569e16bd771d208cbe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733399840857 2024-12-05T11:57:23,945 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 24e0222dd7db42eb972811b658bf35b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399841995 2024-12-05T11:57:23,952 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#C#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:23,952 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/4ab95760172c463eb7845a77cb90caf1 is 50, key is test_row_0/C:col10/1733399842629/Put/seqid=0 2024-12-05T11:57:23,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742069_1245 (size=12207) 2024-12-05T11:57:23,961 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/4ab95760172c463eb7845a77cb90caf1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4ab95760172c463eb7845a77cb90caf1 2024-12-05T11:57:23,967 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/C of d5e0da544abea90f443151cdd8418bf7 into 4ab95760172c463eb7845a77cb90caf1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:23,967 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:23,967 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/C, priority=13, startTime=1733399843882; duration=0sec 2024-12-05T11:57:23,967 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:23,967 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:C 2024-12-05T11:57:24,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T11:57:24,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:24,009 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-05T11:57:24,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/8c48cd5f80994d8d9d3601005146fdd4 is 50, key is test_row_0/A:col10/1733399842648/Put/seqid=0 2024-12-05T11:57:24,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742070_1246 (size=12001) 2024-12-05T11:57:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T11:57:24,352 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/1e9ad491c0b343a8a03715e3d67c1238 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/1e9ad491c0b343a8a03715e3d67c1238 2024-12-05T11:57:24,357 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/A of d5e0da544abea90f443151cdd8418bf7 into 1e9ad491c0b343a8a03715e3d67c1238(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:24,357 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:24,357 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/A, priority=13, startTime=1733399843882; duration=0sec 2024-12-05T11:57:24,357 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:24,357 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:A 2024-12-05T11:57:24,422 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/8c48cd5f80994d8d9d3601005146fdd4 2024-12-05T11:57:24,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/6130d88122cd4c96b07e3b3a7dc9abab is 50, key is test_row_0/B:col10/1733399842648/Put/seqid=0 2024-12-05T11:57:24,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742071_1247 (size=12001) 2024-12-05T11:57:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:24,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:24,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399904812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:24,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399904814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:24,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399904815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:24,833 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/6130d88122cd4c96b07e3b3a7dc9abab 2024-12-05T11:57:24,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/344d33e57b6f4564b68cb943800e6e2f is 50, key is test_row_0/C:col10/1733399842648/Put/seqid=0 2024-12-05T11:57:24,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742072_1248 (size=12001) 2024-12-05T11:57:24,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399904918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:24,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399904927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:24,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399904927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399905037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,039 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in d5e0da544abea90f443151cdd8418bf7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399905047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,049 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in d5e0da544abea90f443151cdd8418bf7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T11:57:25,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399905128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399905137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399905137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,245 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/344d33e57b6f4564b68cb943800e6e2f 2024-12-05T11:57:25,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/8c48cd5f80994d8d9d3601005146fdd4 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8c48cd5f80994d8d9d3601005146fdd4 2024-12-05T11:57:25,253 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8c48cd5f80994d8d9d3601005146fdd4, entries=150, sequenceid=115, filesize=11.7 K 2024-12-05T11:57:25,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/6130d88122cd4c96b07e3b3a7dc9abab as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/6130d88122cd4c96b07e3b3a7dc9abab 2024-12-05T11:57:25,259 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/6130d88122cd4c96b07e3b3a7dc9abab, entries=150, sequenceid=115, filesize=11.7 K 2024-12-05T11:57:25,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/344d33e57b6f4564b68cb943800e6e2f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/344d33e57b6f4564b68cb943800e6e2f 2024-12-05T11:57:25,264 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/344d33e57b6f4564b68cb943800e6e2f, entries=150, sequenceid=115, filesize=11.7 K 2024-12-05T11:57:25,265 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d5e0da544abea90f443151cdd8418bf7 in 1256ms, sequenceid=115, compaction requested=false 2024-12-05T11:57:25,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:25,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:25,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-05T11:57:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-05T11:57:25,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-05T11:57:25,269 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3270 sec 2024-12-05T11:57:25,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.3310 sec 2024-12-05T11:57:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:25,439 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:25,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-05T11:57:25,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/b9b15cdea9d3409a957d221145233d0d is 50, key is test_row_0/A:col10/1733399844814/Put/seqid=0 2024-12-05T11:57:25,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742073_1249 (size=14541) 2024-12-05T11:57:25,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399905460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399905463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399905464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399905568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399905568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399905578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399905777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399905777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399905788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:25,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/b9b15cdea9d3409a957d221145233d0d 2024-12-05T11:57:25,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/d88700818e4249e9921f147a48b6c4f9 is 50, key is test_row_0/B:col10/1733399844814/Put/seqid=0 2024-12-05T11:57:25,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742074_1250 (size=12151) 2024-12-05T11:57:26,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399906088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:26,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399906088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399906098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:26,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/d88700818e4249e9921f147a48b6c4f9 2024-12-05T11:57:26,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/4d0bb05b8f974cc2880551163d02c668 is 50, key is test_row_0/C:col10/1733399844814/Put/seqid=0 2024-12-05T11:57:26,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742075_1251 (size=12151) 2024-12-05T11:57:26,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399906598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:26,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399906598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:26,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399906608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:26,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/4d0bb05b8f974cc2880551163d02c668 2024-12-05T11:57:26,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/b9b15cdea9d3409a957d221145233d0d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/b9b15cdea9d3409a957d221145233d0d 2024-12-05T11:57:26,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/b9b15cdea9d3409a957d221145233d0d, entries=200, sequenceid=134, filesize=14.2 K 2024-12-05T11:57:26,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/d88700818e4249e9921f147a48b6c4f9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/d88700818e4249e9921f147a48b6c4f9 2024-12-05T11:57:26,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/d88700818e4249e9921f147a48b6c4f9, entries=150, sequenceid=134, filesize=11.9 K 2024-12-05T11:57:26,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/4d0bb05b8f974cc2880551163d02c668 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4d0bb05b8f974cc2880551163d02c668 2024-12-05T11:57:26,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4d0bb05b8f974cc2880551163d02c668, entries=150, sequenceid=134, filesize=11.9 K 2024-12-05T11:57:26,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d5e0da544abea90f443151cdd8418bf7 in 1255ms, sequenceid=134, compaction requested=true 2024-12-05T11:57:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:57:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:57:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:57:26,694 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:26,694 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:26,695 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:26,695 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:26,695 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/A is initiating minor compaction (all files) 2024-12-05T11:57:26,695 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/B is initiating minor compaction (all files) 2024-12-05T11:57:26,695 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/B in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:26,695 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/11c714302fd0490e881bfaa70d0535c6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/6130d88122cd4c96b07e3b3a7dc9abab, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/d88700818e4249e9921f147a48b6c4f9] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=35.5 K 2024-12-05T11:57:26,695 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/A in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:26,695 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/1e9ad491c0b343a8a03715e3d67c1238, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8c48cd5f80994d8d9d3601005146fdd4, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/b9b15cdea9d3409a957d221145233d0d] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=37.8 K 2024-12-05T11:57:26,696 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 11c714302fd0490e881bfaa70d0535c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399841995 2024-12-05T11:57:26,696 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e9ad491c0b343a8a03715e3d67c1238, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399841995 2024-12-05T11:57:26,696 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6130d88122cd4c96b07e3b3a7dc9abab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733399842647 2024-12-05T11:57:26,696 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c48cd5f80994d8d9d3601005146fdd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733399842647 2024-12-05T11:57:26,696 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d88700818e4249e9921f147a48b6c4f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399844810 2024-12-05T11:57:26,696 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9b15cdea9d3409a957d221145233d0d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399844810 2024-12-05T11:57:26,703 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#B#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:26,703 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#A#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:26,703 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/d7acf4b159ff4e0888c35af020794ec0 is 50, key is test_row_0/B:col10/1733399844814/Put/seqid=0 2024-12-05T11:57:26,703 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/08331da841014a019ac021976c65f409 is 50, key is test_row_0/A:col10/1733399844814/Put/seqid=0 2024-12-05T11:57:26,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742076_1252 (size=12459) 2024-12-05T11:57:26,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742077_1253 (size=12459) 2024-12-05T11:57:26,724 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/d7acf4b159ff4e0888c35af020794ec0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/d7acf4b159ff4e0888c35af020794ec0 2024-12-05T11:57:26,725 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/08331da841014a019ac021976c65f409 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/08331da841014a019ac021976c65f409 2024-12-05T11:57:26,731 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/B of d5e0da544abea90f443151cdd8418bf7 into d7acf4b159ff4e0888c35af020794ec0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:26,731 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:26,731 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/B, priority=13, startTime=1733399846694; duration=0sec 2024-12-05T11:57:26,731 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:26,731 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:B 2024-12-05T11:57:26,731 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:26,733 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:26,733 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/C is initiating minor compaction (all files) 2024-12-05T11:57:26,733 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/C in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:26,733 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4ab95760172c463eb7845a77cb90caf1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/344d33e57b6f4564b68cb943800e6e2f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4d0bb05b8f974cc2880551163d02c668] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=35.5 K 2024-12-05T11:57:26,733 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ab95760172c463eb7845a77cb90caf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399841995 2024-12-05T11:57:26,734 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 344d33e57b6f4564b68cb943800e6e2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733399842647 2024-12-05T11:57:26,734 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/A of d5e0da544abea90f443151cdd8418bf7 into 08331da841014a019ac021976c65f409(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:26,734 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:26,734 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/A, priority=13, startTime=1733399846694; duration=0sec 2024-12-05T11:57:26,734 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:26,734 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:A 2024-12-05T11:57:26,734 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d0bb05b8f974cc2880551163d02c668, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399844810 2024-12-05T11:57:26,743 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#C#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:26,744 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ef72ab4410ba49fb82bbdf00cb134f5c is 50, key is test_row_0/C:col10/1733399844814/Put/seqid=0 2024-12-05T11:57:26,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742078_1254 (size=12459) 2024-12-05T11:57:26,779 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ef72ab4410ba49fb82bbdf00cb134f5c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ef72ab4410ba49fb82bbdf00cb134f5c 2024-12-05T11:57:26,786 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/C of d5e0da544abea90f443151cdd8418bf7 into ef72ab4410ba49fb82bbdf00cb134f5c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:26,786 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:26,786 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/C, priority=13, startTime=1733399846694; duration=0sec 2024-12-05T11:57:26,786 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:26,786 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:C 2024-12-05T11:57:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T11:57:27,119 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:27,119 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-05T11:57:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T11:57:27,123 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:27,124 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:27,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T11:57:27,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T11:57:27,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:27,279 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-05T11:57:27,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/dd61cbf13f644c22b8f30e6c1a9e9d21 is 50, key is test_row_0/A:col10/1733399845461/Put/seqid=0 2024-12-05T11:57:27,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742079_1255 (size=12151) 2024-12-05T11:57:27,333 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/dd61cbf13f644c22b8f30e6c1a9e9d21 2024-12-05T11:57:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/eaf8537cc75b4cd88ce447d4d04d1975 is 50, key is test_row_0/B:col10/1733399845461/Put/seqid=0 2024-12-05T11:57:27,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742080_1256 (size=12151) 2024-12-05T11:57:27,408 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/eaf8537cc75b4cd88ce447d4d04d1975 2024-12-05T11:57:27,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/cf60c2d46d9f45bc90c0e209b5c7dc6f is 50, key is test_row_0/C:col10/1733399845461/Put/seqid=0 2024-12-05T11:57:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T11:57:27,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742081_1257 (size=12151) 2024-12-05T11:57:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:27,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:27,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399907665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:27,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399907669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:27,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399907669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T11:57:27,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399907777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:27,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399907778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:27,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399907778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:27,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:27,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:27,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:27,878 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/cf60c2d46d9f45bc90c0e209b5c7dc6f 2024-12-05T11:57:27,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/dd61cbf13f644c22b8f30e6c1a9e9d21 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/dd61cbf13f644c22b8f30e6c1a9e9d21 2024-12-05T11:57:27,887 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/dd61cbf13f644c22b8f30e6c1a9e9d21, entries=150, sequenceid=155, filesize=11.9 K 2024-12-05T11:57:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/eaf8537cc75b4cd88ce447d4d04d1975 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/eaf8537cc75b4cd88ce447d4d04d1975 2024-12-05T11:57:27,892 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/eaf8537cc75b4cd88ce447d4d04d1975, entries=150, sequenceid=155, filesize=11.9 K 2024-12-05T11:57:27,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/cf60c2d46d9f45bc90c0e209b5c7dc6f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/cf60c2d46d9f45bc90c0e209b5c7dc6f 2024-12-05T11:57:27,899 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/cf60c2d46d9f45bc90c0e209b5c7dc6f, entries=150, sequenceid=155, filesize=11.9 K 2024-12-05T11:57:27,900 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d5e0da544abea90f443151cdd8418bf7 in 622ms, sequenceid=155, compaction requested=false 2024-12-05T11:57:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-05T11:57:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-05T11:57:27,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-05T11:57:27,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 781 msec 2024-12-05T11:57:27,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 791 msec 2024-12-05T11:57:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:27,991 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:27,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-05T11:57:28,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/c84b10e4401b4de4a4c31a0a2e55bbb7 is 50, key is test_row_0/A:col10/1733399847990/Put/seqid=0 2024-12-05T11:57:28,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399908020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399908025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399908025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742082_1258 (size=12151) 2024-12-05T11:57:28,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/c84b10e4401b4de4a4c31a0a2e55bbb7 2024-12-05T11:57:28,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/1d5f5fc6d1ee494a872f8c63f5c2cdb3 is 50, key is test_row_0/B:col10/1733399847990/Put/seqid=0 2024-12-05T11:57:28,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742083_1259 (size=12151) 2024-12-05T11:57:28,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/1d5f5fc6d1ee494a872f8c63f5c2cdb3 2024-12-05T11:57:28,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/6f4998b931d843a59cfabe43bec749e9 is 50, key is test_row_0/C:col10/1733399847990/Put/seqid=0 2024-12-05T11:57:28,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399908128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399908137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399908137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742084_1260 (size=12151) 2024-12-05T11:57:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/6f4998b931d843a59cfabe43bec749e9 2024-12-05T11:57:28,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/c84b10e4401b4de4a4c31a0a2e55bbb7 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/c84b10e4401b4de4a4c31a0a2e55bbb7 2024-12-05T11:57:28,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/c84b10e4401b4de4a4c31a0a2e55bbb7, entries=150, sequenceid=175, filesize=11.9 K 2024-12-05T11:57:28,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/1d5f5fc6d1ee494a872f8c63f5c2cdb3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/1d5f5fc6d1ee494a872f8c63f5c2cdb3 2024-12-05T11:57:28,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/1d5f5fc6d1ee494a872f8c63f5c2cdb3, entries=150, sequenceid=175, filesize=11.9 K 2024-12-05T11:57:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/6f4998b931d843a59cfabe43bec749e9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/6f4998b931d843a59cfabe43bec749e9 2024-12-05T11:57:28,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/6f4998b931d843a59cfabe43bec749e9, entries=150, sequenceid=175, filesize=11.9 K 2024-12-05T11:57:28,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for d5e0da544abea90f443151cdd8418bf7 in 187ms, sequenceid=175, compaction requested=true 2024-12-05T11:57:28,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:28,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:57:28,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:28,179 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:28,179 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:28,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:57:28,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:28,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:57:28,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:28,180 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:28,180 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/A is initiating minor compaction (all files) 2024-12-05T11:57:28,180 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/A in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:28,181 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/08331da841014a019ac021976c65f409, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/dd61cbf13f644c22b8f30e6c1a9e9d21, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/c84b10e4401b4de4a4c31a0a2e55bbb7] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=35.9 K 2024-12-05T11:57:28,181 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:28,181 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/B is initiating minor compaction (all files) 2024-12-05T11:57:28,181 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/B in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:28,181 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/d7acf4b159ff4e0888c35af020794ec0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/eaf8537cc75b4cd88ce447d4d04d1975, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/1d5f5fc6d1ee494a872f8c63f5c2cdb3] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=35.9 K 2024-12-05T11:57:28,181 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08331da841014a019ac021976c65f409, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399844810 2024-12-05T11:57:28,182 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d7acf4b159ff4e0888c35af020794ec0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399844810 2024-12-05T11:57:28,182 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd61cbf13f644c22b8f30e6c1a9e9d21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733399845456 2024-12-05T11:57:28,183 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting eaf8537cc75b4cd88ce447d4d04d1975, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733399845456 2024-12-05T11:57:28,183 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c84b10e4401b4de4a4c31a0a2e55bbb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733399847662 2024-12-05T11:57:28,184 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d5f5fc6d1ee494a872f8c63f5c2cdb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733399847662 2024-12-05T11:57:28,197 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:28,197 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/f9847076e71e47b6a01a51ae44e7e408 is 50, key is test_row_0/B:col10/1733399847990/Put/seqid=0 2024-12-05T11:57:28,200 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#A#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:28,201 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/09177793fae4475a879d45321177e95c is 50, key is test_row_0/A:col10/1733399847990/Put/seqid=0 2024-12-05T11:57:28,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742086_1262 (size=12561) 2024-12-05T11:57:28,230 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/09177793fae4475a879d45321177e95c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/09177793fae4475a879d45321177e95c 2024-12-05T11:57:28,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742085_1261 (size=12561) 2024-12-05T11:57:28,243 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/A of d5e0da544abea90f443151cdd8418bf7 into 09177793fae4475a879d45321177e95c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:28,243 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:28,243 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/A, priority=13, startTime=1733399848178; duration=0sec 2024-12-05T11:57:28,243 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:28,243 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:A 2024-12-05T11:57:28,243 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:28,247 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:28,247 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/C is initiating minor compaction (all files) 2024-12-05T11:57:28,247 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/C in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:28,247 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ef72ab4410ba49fb82bbdf00cb134f5c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/cf60c2d46d9f45bc90c0e209b5c7dc6f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/6f4998b931d843a59cfabe43bec749e9] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=35.9 K 2024-12-05T11:57:28,248 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef72ab4410ba49fb82bbdf00cb134f5c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733399844810 2024-12-05T11:57:28,248 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf60c2d46d9f45bc90c0e209b5c7dc6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733399845456 2024-12-05T11:57:28,249 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f4998b931d843a59cfabe43bec749e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733399847662 2024-12-05T11:57:28,252 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/f9847076e71e47b6a01a51ae44e7e408 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f9847076e71e47b6a01a51ae44e7e408 2024-12-05T11:57:28,260 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#C#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:28,261 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/B of d5e0da544abea90f443151cdd8418bf7 into f9847076e71e47b6a01a51ae44e7e408(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:28,261 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:28,261 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/B, priority=13, startTime=1733399848179; duration=0sec 2024-12-05T11:57:28,261 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/1bd74d0b6d444c05831d5afd7b78bd21 is 50, key is test_row_0/C:col10/1733399847990/Put/seqid=0 2024-12-05T11:57:28,261 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:28,261 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:B 2024-12-05T11:57:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T11:57:28,268 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:28,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-05T11:57:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:57:28,271 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:28,273 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:28,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:28,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742087_1263 (size=12561) 2024-12-05T11:57:28,288 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/1bd74d0b6d444c05831d5afd7b78bd21 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/1bd74d0b6d444c05831d5afd7b78bd21 2024-12-05T11:57:28,294 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/C of d5e0da544abea90f443151cdd8418bf7 into 1bd74d0b6d444c05831d5afd7b78bd21(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:28,294 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:28,294 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/C, priority=13, startTime=1733399848179; duration=0sec 2024-12-05T11:57:28,295 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:28,295 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:C 2024-12-05T11:57:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:28,340 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:28,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-05T11:57:28,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/d538e8fe54c44f31880233da16160c61 is 50, key is test_row_0/A:col10/1733399848024/Put/seqid=0 2024-12-05T11:57:28,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742088_1264 (size=14541) 2024-12-05T11:57:28,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/d538e8fe54c44f31880233da16160c61 2024-12-05T11:57:28,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399908365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399908365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399908369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/4320a7a9000842d0b9656e1697beee9c is 50, key is test_row_0/B:col10/1733399848024/Put/seqid=0 2024-12-05T11:57:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:57:28,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742089_1265 (size=12151) 2024-12-05T11:57:28,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/4320a7a9000842d0b9656e1697beee9c 2024-12-05T11:57:28,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/4e76f8e68e8f452e806cd1adeebd03a2 is 50, key is test_row_0/C:col10/1733399848024/Put/seqid=0 2024-12-05T11:57:28,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742090_1266 (size=12151) 2024-12-05T11:57:28,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:57:28,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:28,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:28,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:28,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:28,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:28,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399908478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399908478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399908478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:57:28,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:28,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:28,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:28,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:28,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:57:28,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399908687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399908688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399908688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:28,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:28,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:57:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:28,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:28,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/4e76f8e68e8f452e806cd1adeebd03a2 2024-12-05T11:57:28,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/d538e8fe54c44f31880233da16160c61 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d538e8fe54c44f31880233da16160c61 2024-12-05T11:57:28,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d538e8fe54c44f31880233da16160c61, entries=200, sequenceid=196, filesize=14.2 K 2024-12-05T11:57:28,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/4320a7a9000842d0b9656e1697beee9c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/4320a7a9000842d0b9656e1697beee9c 2024-12-05T11:57:28,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/4320a7a9000842d0b9656e1697beee9c, entries=150, sequenceid=196, filesize=11.9 K 2024-12-05T11:57:28,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/4e76f8e68e8f452e806cd1adeebd03a2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4e76f8e68e8f452e806cd1adeebd03a2 2024-12-05T11:57:28,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4e76f8e68e8f452e806cd1adeebd03a2, entries=150, sequenceid=196, filesize=11.9 K 2024-12-05T11:57:28,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d5e0da544abea90f443151cdd8418bf7 in 487ms, sequenceid=196, compaction requested=false 2024-12-05T11:57:28,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:28,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:57:28,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:28,884 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-05T11:57:28,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/83030c555b514a28963e5a674859e0b3 is 50, key is test_row_0/A:col10/1733399848363/Put/seqid=0 2024-12-05T11:57:28,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742091_1267 (size=12151) 2024-12-05T11:57:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:57:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:29,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:29,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399909012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399909012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399909015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399909058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,060 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in d5e0da544abea90f443151cdd8418bf7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399909097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,099 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in d5e0da544abea90f443151cdd8418bf7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399909118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399909118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399909128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,297 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/83030c555b514a28963e5a674859e0b3 2024-12-05T11:57:29,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/5bdec0b3048a4812881720a76ad33550 is 50, key is test_row_0/B:col10/1733399848363/Put/seqid=0 2024-12-05T11:57:29,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742092_1268 (size=12151) 2024-12-05T11:57:29,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399909327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399909327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399909338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:57:29,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399909638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399909638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399909647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:29,707 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/5bdec0b3048a4812881720a76ad33550 2024-12-05T11:57:29,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ce9fe20c27414eb5a77ba4e4f143d225 is 50, key is test_row_0/C:col10/1733399848363/Put/seqid=0 2024-12-05T11:57:29,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742093_1269 (size=12151) 2024-12-05T11:57:29,718 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ce9fe20c27414eb5a77ba4e4f143d225 2024-12-05T11:57:29,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/83030c555b514a28963e5a674859e0b3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/83030c555b514a28963e5a674859e0b3 2024-12-05T11:57:29,726 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/83030c555b514a28963e5a674859e0b3, entries=150, sequenceid=214, filesize=11.9 K 2024-12-05T11:57:29,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/5bdec0b3048a4812881720a76ad33550 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/5bdec0b3048a4812881720a76ad33550 2024-12-05T11:57:29,730 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/5bdec0b3048a4812881720a76ad33550, entries=150, sequenceid=214, filesize=11.9 K 2024-12-05T11:57:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ce9fe20c27414eb5a77ba4e4f143d225 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ce9fe20c27414eb5a77ba4e4f143d225 2024-12-05T11:57:29,735 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ce9fe20c27414eb5a77ba4e4f143d225, entries=150, sequenceid=214, filesize=11.9 K 2024-12-05T11:57:29,736 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d5e0da544abea90f443151cdd8418bf7 in 852ms, sequenceid=214, compaction requested=true 2024-12-05T11:57:29,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:29,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:29,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-05T11:57:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-05T11:57:29,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-05T11:57:29,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4650 sec 2024-12-05T11:57:29,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4720 sec 2024-12-05T11:57:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:30,150 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:30,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-05T11:57:30,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/afe7d523a8324ad4bb42a88d024ae2a9 is 50, key is test_row_0/A:col10/1733399850149/Put/seqid=0 2024-12-05T11:57:30,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742094_1270 (size=16931) 2024-12-05T11:57:30,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399910168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:30,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399910170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:30,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399910170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:30,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399910277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:30,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399910277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399910277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:57:30,418 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:30,418 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-05T11:57:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T11:57:30,420 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:30,420 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:30,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:30,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399910487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:30,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399910487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:30,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399910487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T11:57:30,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/afe7d523a8324ad4bb42a88d024ae2a9 2024-12-05T11:57:30,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T11:57:30,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:30,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/1267570cad91423088125a387bac9856 is 50, key is test_row_0/B:col10/1733399850149/Put/seqid=0 2024-12-05T11:57:30,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:30,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:30,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:30,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:30,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742095_1271 (size=12151) 2024-12-05T11:57:30,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T11:57:30,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:30,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:30,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:30,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:30,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T11:57:30,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399910798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:30,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:30,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399910798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399910798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:30,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T11:57:30,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:30,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:30,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:30,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:30,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:30,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/1267570cad91423088125a387bac9856 2024-12-05T11:57:30,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/18081e3726c34fb59bdcca91d8b7d252 is 50, key is test_row_0/C:col10/1733399850149/Put/seqid=0 2024-12-05T11:57:31,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742096_1272 (size=12151) 2024-12-05T11:57:31,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/18081e3726c34fb59bdcca91d8b7d252 2024-12-05T11:57:31,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/afe7d523a8324ad4bb42a88d024ae2a9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/afe7d523a8324ad4bb42a88d024ae2a9 2024-12-05T11:57:31,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/afe7d523a8324ad4bb42a88d024ae2a9, entries=250, sequenceid=234, filesize=16.5 K 2024-12-05T11:57:31,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/1267570cad91423088125a387bac9856 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/1267570cad91423088125a387bac9856 2024-12-05T11:57:31,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/1267570cad91423088125a387bac9856, entries=150, sequenceid=234, filesize=11.9 K 2024-12-05T11:57:31,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/18081e3726c34fb59bdcca91d8b7d252 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/18081e3726c34fb59bdcca91d8b7d252 2024-12-05T11:57:31,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/18081e3726c34fb59bdcca91d8b7d252, entries=150, sequenceid=234, filesize=11.9 K 2024-12-05T11:57:31,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d5e0da544abea90f443151cdd8418bf7 in 873ms, sequenceid=234, compaction requested=true 2024-12-05T11:57:31,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:31,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:57:31,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:31,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:57:31,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:31,023 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:57:31,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:57:31,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:31,023 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:57:31,024 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56184 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:57:31,024 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:57:31,024 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/A is initiating minor compaction (all files) 2024-12-05T11:57:31,024 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/B is initiating minor compaction (all files) 2024-12-05T11:57:31,024 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/A in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:31,024 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/B in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:31,024 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/09177793fae4475a879d45321177e95c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d538e8fe54c44f31880233da16160c61, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/83030c555b514a28963e5a674859e0b3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/afe7d523a8324ad4bb42a88d024ae2a9] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=54.9 K 2024-12-05T11:57:31,024 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f9847076e71e47b6a01a51ae44e7e408, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/4320a7a9000842d0b9656e1697beee9c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/5bdec0b3048a4812881720a76ad33550, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/1267570cad91423088125a387bac9856] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=47.9 K 2024-12-05T11:57:31,025 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting f9847076e71e47b6a01a51ae44e7e408, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733399847662 2024-12-05T11:57:31,025 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09177793fae4475a879d45321177e95c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733399847662 2024-12-05T11:57:31,025 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4320a7a9000842d0b9656e1697beee9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733399848023 2024-12-05T11:57:31,025 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d538e8fe54c44f31880233da16160c61, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733399848010 2024-12-05T11:57:31,025 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bdec0b3048a4812881720a76ad33550, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733399848353 2024-12-05T11:57:31,025 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83030c555b514a28963e5a674859e0b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733399848353 2024-12-05T11:57:31,025 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1267570cad91423088125a387bac9856, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733399849010 2024-12-05T11:57:31,025 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting afe7d523a8324ad4bb42a88d024ae2a9, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733399849010 2024-12-05T11:57:31,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T11:57:31,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:31,033 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-05T11:57:31,035 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:31,036 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/78696ff61e3f4f0daf2b78262d37403d is 50, key is test_row_0/B:col10/1733399850149/Put/seqid=0 2024-12-05T11:57:31,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/c399a258a23d4f5f9b3e545009836826 is 50, key is test_row_0/A:col10/1733399850167/Put/seqid=0 2024-12-05T11:57:31,039 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#A#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:31,039 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/cf98f3ac0f1f42529628fc477e220361 is 50, key is test_row_0/A:col10/1733399850149/Put/seqid=0 2024-12-05T11:57:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T11:57:31,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742097_1273 (size=12697) 2024-12-05T11:57:31,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742099_1275 (size=12697) 2024-12-05T11:57:31,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742098_1274 (size=12151) 2024-12-05T11:57:31,065 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/78696ff61e3f4f0daf2b78262d37403d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/78696ff61e3f4f0daf2b78262d37403d 2024-12-05T11:57:31,066 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/c399a258a23d4f5f9b3e545009836826 2024-12-05T11:57:31,070 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/B of d5e0da544abea90f443151cdd8418bf7 into 78696ff61e3f4f0daf2b78262d37403d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:31,070 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:31,070 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/B, priority=12, startTime=1733399851023; duration=0sec 2024-12-05T11:57:31,070 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:31,070 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:B 2024-12-05T11:57:31,070 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:57:31,072 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:57:31,072 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/C is initiating minor compaction (all files) 2024-12-05T11:57:31,072 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/C in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:31,072 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/1bd74d0b6d444c05831d5afd7b78bd21, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4e76f8e68e8f452e806cd1adeebd03a2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ce9fe20c27414eb5a77ba4e4f143d225, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/18081e3726c34fb59bdcca91d8b7d252] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=47.9 K 2024-12-05T11:57:31,073 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd74d0b6d444c05831d5afd7b78bd21, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733399847662 2024-12-05T11:57:31,073 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e76f8e68e8f452e806cd1adeebd03a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733399848023 2024-12-05T11:57:31,073 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ce9fe20c27414eb5a77ba4e4f143d225, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733399848353 2024-12-05T11:57:31,073 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 18081e3726c34fb59bdcca91d8b7d252, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733399849010 2024-12-05T11:57:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/c1827ebe565147b58b4bfa6e3a5c0362 is 50, key is test_row_0/B:col10/1733399850167/Put/seqid=0 2024-12-05T11:57:31,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742100_1276 (size=12151) 2024-12-05T11:57:31,080 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/c1827ebe565147b58b4bfa6e3a5c0362 2024-12-05T11:57:31,083 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#C#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:31,084 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/353a71cf60b64e89a2947fb9caccfcf9 is 50, key is test_row_0/C:col10/1733399850149/Put/seqid=0 2024-12-05T11:57:31,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/28bd33a308c144459970a1128f37a0cf is 50, key is test_row_0/C:col10/1733399850167/Put/seqid=0 2024-12-05T11:57:31,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742101_1277 (size=12697) 2024-12-05T11:57:31,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742102_1278 (size=12151) 2024-12-05T11:57:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:31,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:31,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399911326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399911330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399911330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399911438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399911438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399911438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,465 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/cf98f3ac0f1f42529628fc477e220361 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/cf98f3ac0f1f42529628fc477e220361 2024-12-05T11:57:31,470 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/A of d5e0da544abea90f443151cdd8418bf7 into cf98f3ac0f1f42529628fc477e220361(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:31,470 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:31,470 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/A, priority=12, startTime=1733399851023; duration=0sec 2024-12-05T11:57:31,470 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:31,470 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:A 2024-12-05T11:57:31,493 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/28bd33a308c144459970a1128f37a0cf 2024-12-05T11:57:31,495 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/353a71cf60b64e89a2947fb9caccfcf9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/353a71cf60b64e89a2947fb9caccfcf9 2024-12-05T11:57:31,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/c399a258a23d4f5f9b3e545009836826 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/c399a258a23d4f5f9b3e545009836826 2024-12-05T11:57:31,500 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/C of d5e0da544abea90f443151cdd8418bf7 into 353a71cf60b64e89a2947fb9caccfcf9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:31,500 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:31,500 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/C, priority=12, startTime=1733399851023; duration=0sec 2024-12-05T11:57:31,500 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:31,501 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:C 2024-12-05T11:57:31,501 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/c399a258a23d4f5f9b3e545009836826, entries=150, sequenceid=250, filesize=11.9 K 2024-12-05T11:57:31,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/c1827ebe565147b58b4bfa6e3a5c0362 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/c1827ebe565147b58b4bfa6e3a5c0362 2024-12-05T11:57:31,505 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/c1827ebe565147b58b4bfa6e3a5c0362, entries=150, sequenceid=250, filesize=11.9 K 2024-12-05T11:57:31,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/28bd33a308c144459970a1128f37a0cf as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/28bd33a308c144459970a1128f37a0cf 2024-12-05T11:57:31,509 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/28bd33a308c144459970a1128f37a0cf, entries=150, sequenceid=250, filesize=11.9 K 2024-12-05T11:57:31,510 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d5e0da544abea90f443151cdd8418bf7 in 477ms, sequenceid=250, compaction requested=false 2024-12-05T11:57:31,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:31,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:31,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-05T11:57:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-05T11:57:31,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-05T11:57:31,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0910 sec 2024-12-05T11:57:31,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0960 sec 2024-12-05T11:57:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T11:57:31,558 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:31,558 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-05T11:57:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T11:57:31,560 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:31,561 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:31,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:31,650 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:31,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-05T11:57:31,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/120a84ba7d1949998bf9cc7250c92d85 is 50, key is test_row_0/A:col10/1733399851649/Put/seqid=0 2024-12-05T11:57:31,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742103_1279 (size=14741) 2024-12-05T11:57:31,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399911662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399911662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399911664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T11:57:31,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T11:57:31,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:31,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:31,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:31,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:31,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:31,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399911768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399911768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399911777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T11:57:31,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:31,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:31,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:31,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:31,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:31,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:31,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T11:57:31,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399911978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399911978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:31,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:31,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399911987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:31,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:31,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:32,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T11:57:32,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:32,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:32,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:32,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:32,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:32,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/120a84ba7d1949998bf9cc7250c92d85 2024-12-05T11:57:32,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/f4d423f1d90c4b4081534bc8756f0f0c is 50, key is test_row_0/B:col10/1733399851649/Put/seqid=0 2024-12-05T11:57:32,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742104_1280 (size=12301) 2024-12-05T11:57:32,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/f4d423f1d90c4b4081534bc8756f0f0c 2024-12-05T11:57:32,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/585d8e93e9d14c8c8c69cf6685b2dbd7 is 50, key is test_row_0/C:col10/1733399851649/Put/seqid=0 2024-12-05T11:57:32,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742105_1281 (size=12301) 2024-12-05T11:57:32,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/585d8e93e9d14c8c8c69cf6685b2dbd7 2024-12-05T11:57:32,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/120a84ba7d1949998bf9cc7250c92d85 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/120a84ba7d1949998bf9cc7250c92d85 2024-12-05T11:57:32,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/120a84ba7d1949998bf9cc7250c92d85, entries=200, sequenceid=276, filesize=14.4 K 2024-12-05T11:57:32,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/f4d423f1d90c4b4081534bc8756f0f0c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f4d423f1d90c4b4081534bc8756f0f0c 2024-12-05T11:57:32,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f4d423f1d90c4b4081534bc8756f0f0c, entries=150, sequenceid=276, filesize=12.0 K 2024-12-05T11:57:32,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/585d8e93e9d14c8c8c69cf6685b2dbd7 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/585d8e93e9d14c8c8c69cf6685b2dbd7 2024-12-05T11:57:32,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/585d8e93e9d14c8c8c69cf6685b2dbd7, entries=150, sequenceid=276, filesize=12.0 K 2024-12-05T11:57:32,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d5e0da544abea90f443151cdd8418bf7 in 477ms, sequenceid=276, compaction requested=true 2024-12-05T11:57:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:57:32,128 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:57:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:32,128 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:57:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:32,129 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:32,129 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/A is initiating minor compaction (all files) 2024-12-05T11:57:32,129 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/A in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:32,129 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/cf98f3ac0f1f42529628fc477e220361, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/c399a258a23d4f5f9b3e545009836826, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/120a84ba7d1949998bf9cc7250c92d85] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=38.7 K 2024-12-05T11:57:32,129 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:32,129 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/B is initiating minor compaction (all files) 2024-12-05T11:57:32,129 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/B in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:32,129 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/78696ff61e3f4f0daf2b78262d37403d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/c1827ebe565147b58b4bfa6e3a5c0362, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f4d423f1d90c4b4081534bc8756f0f0c] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=36.3 K 2024-12-05T11:57:32,130 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf98f3ac0f1f42529628fc477e220361, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733399849010 2024-12-05T11:57:32,130 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 78696ff61e3f4f0daf2b78262d37403d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733399849010 2024-12-05T11:57:32,130 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c399a258a23d4f5f9b3e545009836826, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733399850163 2024-12-05T11:57:32,130 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c1827ebe565147b58b4bfa6e3a5c0362, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733399850163 2024-12-05T11:57:32,130 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 120a84ba7d1949998bf9cc7250c92d85, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733399851317 2024-12-05T11:57:32,131 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d423f1d90c4b4081534bc8756f0f0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733399851649 2024-12-05T11:57:32,143 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#A#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:32,143 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/e31c23f9fc604afea3e0bfd91cd66968 is 50, key is test_row_0/A:col10/1733399851649/Put/seqid=0 2024-12-05T11:57:32,145 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#B#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:32,145 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/b948b27193f442d7b45bbd53edea40ab is 50, key is test_row_0/B:col10/1733399851649/Put/seqid=0 2024-12-05T11:57:32,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742106_1282 (size=12949) 2024-12-05T11:57:32,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742107_1283 (size=12949) 2024-12-05T11:57:32,168 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/e31c23f9fc604afea3e0bfd91cd66968 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/e31c23f9fc604afea3e0bfd91cd66968 2024-12-05T11:57:32,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T11:57:32,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:32,173 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T11:57:32,174 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/A of d5e0da544abea90f443151cdd8418bf7 into e31c23f9fc604afea3e0bfd91cd66968(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:32,174 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:32,174 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/A, priority=13, startTime=1733399852128; duration=0sec 2024-12-05T11:57:32,174 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:32,174 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:A 2024-12-05T11:57:32,174 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:32,175 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:32,175 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/C is initiating minor compaction (all files) 2024-12-05T11:57:32,175 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/C in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:32,176 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/353a71cf60b64e89a2947fb9caccfcf9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/28bd33a308c144459970a1128f37a0cf, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/585d8e93e9d14c8c8c69cf6685b2dbd7] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=36.3 K 2024-12-05T11:57:32,176 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 353a71cf60b64e89a2947fb9caccfcf9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733399849010 2024-12-05T11:57:32,176 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28bd33a308c144459970a1128f37a0cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733399850163 2024-12-05T11:57:32,177 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 585d8e93e9d14c8c8c69cf6685b2dbd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733399851649 2024-12-05T11:57:32,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/835439ef76aa47259a8fc28951290b38 is 50, key is test_row_0/A:col10/1733399851660/Put/seqid=0 2024-12-05T11:57:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T11:57:32,189 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:32,189 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/dbac7ce1468c4ce6b6dd82111c188f5b is 50, key is test_row_0/C:col10/1733399851649/Put/seqid=0 2024-12-05T11:57:32,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742108_1284 (size=12301) 2024-12-05T11:57:32,197 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/835439ef76aa47259a8fc28951290b38 2024-12-05T11:57:32,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742109_1285 (size=12949) 2024-12-05T11:57:32,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/fb45c341671a4cebb3298258b1bcfa6d is 50, key is test_row_0/B:col10/1733399851660/Put/seqid=0 2024-12-05T11:57:32,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742110_1286 (size=12301) 2024-12-05T11:57:32,209 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/dbac7ce1468c4ce6b6dd82111c188f5b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/dbac7ce1468c4ce6b6dd82111c188f5b 2024-12-05T11:57:32,214 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/C of d5e0da544abea90f443151cdd8418bf7 into dbac7ce1468c4ce6b6dd82111c188f5b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:32,214 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:32,214 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/C, priority=13, startTime=1733399852128; duration=0sec 2024-12-05T11:57:32,214 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:32,214 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:C 2024-12-05T11:57:32,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:32,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:32,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399912314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:32,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:32,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:32,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399912314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:32,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:32,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:32,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399912315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:32,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399912417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399912417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:32,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399912428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:32,568 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/b948b27193f442d7b45bbd53edea40ab as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b948b27193f442d7b45bbd53edea40ab 2024-12-05T11:57:32,572 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/B of d5e0da544abea90f443151cdd8418bf7 into b948b27193f442d7b45bbd53edea40ab(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:32,572 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:32,572 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/B, priority=13, startTime=1733399852128; duration=0sec 2024-12-05T11:57:32,572 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:32,572 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:B 2024-12-05T11:57:32,610 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/fb45c341671a4cebb3298258b1bcfa6d 2024-12-05T11:57:32,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/078f007f842f40c7b8178d38aca441ca is 50, key is test_row_0/C:col10/1733399851660/Put/seqid=0 2024-12-05T11:57:32,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742111_1287 (size=12301) 2024-12-05T11:57:32,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399912628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:32,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399912628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:32,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399912637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T11:57:32,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399912938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:32,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399912938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:32,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399912948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:32,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:33,020 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/078f007f842f40c7b8178d38aca441ca 2024-12-05T11:57:33,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/835439ef76aa47259a8fc28951290b38 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/835439ef76aa47259a8fc28951290b38 2024-12-05T11:57:33,028 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/835439ef76aa47259a8fc28951290b38, entries=150, sequenceid=290, filesize=12.0 K 2024-12-05T11:57:33,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/fb45c341671a4cebb3298258b1bcfa6d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/fb45c341671a4cebb3298258b1bcfa6d 2024-12-05T11:57:33,032 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/fb45c341671a4cebb3298258b1bcfa6d, entries=150, sequenceid=290, filesize=12.0 K 2024-12-05T11:57:33,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/078f007f842f40c7b8178d38aca441ca as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/078f007f842f40c7b8178d38aca441ca 2024-12-05T11:57:33,037 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/078f007f842f40c7b8178d38aca441ca, entries=150, sequenceid=290, filesize=12.0 K 2024-12-05T11:57:33,038 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d5e0da544abea90f443151cdd8418bf7 in 865ms, sequenceid=290, compaction requested=false 2024-12-05T11:57:33,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:33,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:33,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-05T11:57:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-05T11:57:33,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-05T11:57:33,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-12-05T11:57:33,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4850 sec 2024-12-05T11:57:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:33,451 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:33,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T11:57:33,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/d3be594ccc0e4937b5272d84ebf24353 is 50, key is test_row_0/A:col10/1733399853449/Put/seqid=0 2024-12-05T11:57:33,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742112_1288 (size=17181) 2024-12-05T11:57:33,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399913462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:33,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:33,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399913463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:33,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399913464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:33,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399913568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:33,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399913578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:33,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399913578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T11:57:33,708 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:33,709 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-05T11:57:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T11:57:33,710 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:33,711 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:33,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:33,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399913778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:33,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399913788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:33,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399913788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T11:57:33,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/d3be594ccc0e4937b5272d84ebf24353 2024-12-05T11:57:33,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-05T11:57:33,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:33,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:33,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:33,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:33,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:33,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/964563b9c683496d8b1770b0f0a8f947 is 50, key is test_row_0/B:col10/1733399853449/Put/seqid=0 2024-12-05T11:57:33,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742113_1289 (size=12301) 2024-12-05T11:57:34,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-05T11:57:34,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:34,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:34,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:34,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:34,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T11:57:34,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399914087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:34,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399914098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:34,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399914098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:34,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-05T11:57:34,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:34,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:34,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:34,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:34,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:34,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:34,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/964563b9c683496d8b1770b0f0a8f947 2024-12-05T11:57:34,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ca76f836db88491fbe6a3d186a6b1d6b is 50, key is test_row_0/C:col10/1733399853449/Put/seqid=0 2024-12-05T11:57:34,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742114_1290 (size=12301) 2024-12-05T11:57:34,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ca76f836db88491fbe6a3d186a6b1d6b 2024-12-05T11:57:34,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/d3be594ccc0e4937b5272d84ebf24353 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d3be594ccc0e4937b5272d84ebf24353 2024-12-05T11:57:34,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d3be594ccc0e4937b5272d84ebf24353, entries=250, sequenceid=317, filesize=16.8 K 2024-12-05T11:57:34,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/964563b9c683496d8b1770b0f0a8f947 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/964563b9c683496d8b1770b0f0a8f947 2024-12-05T11:57:34,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/964563b9c683496d8b1770b0f0a8f947, entries=150, sequenceid=317, filesize=12.0 K 2024-12-05T11:57:34,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ca76f836db88491fbe6a3d186a6b1d6b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ca76f836db88491fbe6a3d186a6b1d6b 2024-12-05T11:57:34,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ca76f836db88491fbe6a3d186a6b1d6b, entries=150, sequenceid=317, filesize=12.0 K 2024-12-05T11:57:34,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d5e0da544abea90f443151cdd8418bf7 in 847ms, sequenceid=317, compaction requested=true 2024-12-05T11:57:34,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:34,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:57:34,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:34,298 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:34,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:57:34,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:34,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:57:34,298 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:34,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:34,299 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:34,299 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:34,299 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/B is initiating minor compaction (all files) 2024-12-05T11:57:34,299 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/A is initiating minor compaction (all files) 2024-12-05T11:57:34,299 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/A in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:34,299 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/B in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:34,299 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/e31c23f9fc604afea3e0bfd91cd66968, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/835439ef76aa47259a8fc28951290b38, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d3be594ccc0e4937b5272d84ebf24353] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=41.4 K 2024-12-05T11:57:34,299 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b948b27193f442d7b45bbd53edea40ab, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/fb45c341671a4cebb3298258b1bcfa6d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/964563b9c683496d8b1770b0f0a8f947] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=36.7 K 2024-12-05T11:57:34,299 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e31c23f9fc604afea3e0bfd91cd66968, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733399851649 2024-12-05T11:57:34,299 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting b948b27193f442d7b45bbd53edea40ab, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733399851649 2024-12-05T11:57:34,300 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 835439ef76aa47259a8fc28951290b38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733399851655 2024-12-05T11:57:34,300 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting fb45c341671a4cebb3298258b1bcfa6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733399851655 2024-12-05T11:57:34,300 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3be594ccc0e4937b5272d84ebf24353, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733399852300 2024-12-05T11:57:34,300 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 964563b9c683496d8b1770b0f0a8f947, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733399852313 2024-12-05T11:57:34,307 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:34,307 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:34,307 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/f7500c7d397143799118234ca604a546 is 50, key is test_row_0/A:col10/1733399853449/Put/seqid=0 2024-12-05T11:57:34,307 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/55485a13f532415c9c209690846736f2 is 50, key is test_row_0/B:col10/1733399853449/Put/seqid=0 2024-12-05T11:57:34,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-05T11:57:34,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:34,327 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T11:57:34,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742115_1291 (size=13051) 2024-12-05T11:57:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T11:57:34,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742116_1292 (size=13051) 2024-12-05T11:57:34,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/6c687c2670fa48de81330114222a4105 is 50, key is test_row_0/A:col10/1733399853461/Put/seqid=0 2024-12-05T11:57:34,347 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/55485a13f532415c9c209690846736f2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/55485a13f532415c9c209690846736f2 2024-12-05T11:57:34,353 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/B of d5e0da544abea90f443151cdd8418bf7 into 55485a13f532415c9c209690846736f2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:34,353 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:34,353 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/B, priority=13, startTime=1733399854298; duration=0sec 2024-12-05T11:57:34,353 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:34,353 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:B 2024-12-05T11:57:34,353 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:34,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742117_1293 (size=12301) 2024-12-05T11:57:34,355 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:34,355 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/C is initiating minor compaction (all files) 2024-12-05T11:57:34,355 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/C in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:34,355 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/dbac7ce1468c4ce6b6dd82111c188f5b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/078f007f842f40c7b8178d38aca441ca, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ca76f836db88491fbe6a3d186a6b1d6b] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=36.7 K 2024-12-05T11:57:34,355 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/6c687c2670fa48de81330114222a4105 2024-12-05T11:57:34,356 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting dbac7ce1468c4ce6b6dd82111c188f5b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733399851649 2024-12-05T11:57:34,356 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 078f007f842f40c7b8178d38aca441ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733399851655 2024-12-05T11:57:34,356 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ca76f836db88491fbe6a3d186a6b1d6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733399852313 2024-12-05T11:57:34,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/0dc9cfcc594349a489075d983006cacb is 50, key is test_row_0/B:col10/1733399853461/Put/seqid=0 2024-12-05T11:57:34,365 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:34,366 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ee887bb26aab4dfc9a4aba860a3085eb is 50, key is test_row_0/C:col10/1733399853449/Put/seqid=0 2024-12-05T11:57:34,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742118_1294 (size=12301) 2024-12-05T11:57:34,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742119_1295 (size=13051) 2024-12-05T11:57:34,375 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/ee887bb26aab4dfc9a4aba860a3085eb as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ee887bb26aab4dfc9a4aba860a3085eb 2024-12-05T11:57:34,381 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/C of d5e0da544abea90f443151cdd8418bf7 into ee887bb26aab4dfc9a4aba860a3085eb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:34,382 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:34,382 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/C, priority=13, startTime=1733399854298; duration=0sec 2024-12-05T11:57:34,382 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:34,382 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:C 2024-12-05T11:57:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:34,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:34,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399914626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:34,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399914628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:34,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:34,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399914628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:34,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:34,734 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/f7500c7d397143799118234ca604a546 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/f7500c7d397143799118234ca604a546 2024-12-05T11:57:34,738 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/A of d5e0da544abea90f443151cdd8418bf7 into f7500c7d397143799118234ca604a546(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:34,738 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:34,738 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/A, priority=13, startTime=1733399854298; duration=0sec 2024-12-05T11:57:34,738 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:34,738 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:A 2024-12-05T11:57:34,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399914737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:34,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399914737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:34,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399914738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:34,769 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/0dc9cfcc594349a489075d983006cacb 2024-12-05T11:57:34,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/902cb488d9014f7d9fe7001b0601116b is 50, key is test_row_0/C:col10/1733399853461/Put/seqid=0 2024-12-05T11:57:34,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742120_1296 (size=12301) 2024-12-05T11:57:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T11:57:34,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399914947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:34,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399914947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:34,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399914947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,196 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/902cb488d9014f7d9fe7001b0601116b 2024-12-05T11:57:35,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/6c687c2670fa48de81330114222a4105 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/6c687c2670fa48de81330114222a4105 2024-12-05T11:57:35,203 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/6c687c2670fa48de81330114222a4105, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T11:57:35,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/0dc9cfcc594349a489075d983006cacb as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/0dc9cfcc594349a489075d983006cacb 2024-12-05T11:57:35,208 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/0dc9cfcc594349a489075d983006cacb, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T11:57:35,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/902cb488d9014f7d9fe7001b0601116b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/902cb488d9014f7d9fe7001b0601116b 2024-12-05T11:57:35,212 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/902cb488d9014f7d9fe7001b0601116b, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T11:57:35,213 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d5e0da544abea90f443151cdd8418bf7 in 886ms, sequenceid=328, compaction requested=false 2024-12-05T11:57:35,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:35,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:35,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-05T11:57:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-05T11:57:35,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-05T11:57:35,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5030 sec 2024-12-05T11:57:35,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.5070 sec 2024-12-05T11:57:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:35,259 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:35,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-05T11:57:35,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/69dbbf2fc5b64d9e83893c4e91cc34c2 is 50, key is test_row_0/A:col10/1733399854627/Put/seqid=0 2024-12-05T11:57:35,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742121_1297 (size=14741) 2024-12-05T11:57:35,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399915265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:35,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399915265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399915269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399915378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:35,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399915378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399915378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399915588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:35,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399915588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399915588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/69dbbf2fc5b64d9e83893c4e91cc34c2 2024-12-05T11:57:35,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/6aef96b3140d4fc8a3ae01ba0a79cbc7 is 50, key is test_row_0/B:col10/1733399854627/Put/seqid=0 2024-12-05T11:57:35,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742122_1298 (size=12301) 2024-12-05T11:57:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T11:57:35,858 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:35,859 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-05T11:57:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T11:57:35,860 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:35,861 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:35,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:35,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399915897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:35,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:35,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399915897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:35,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399915897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T11:57:36,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T11:57:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:36,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:36,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/6aef96b3140d4fc8a3ae01ba0a79cbc7 2024-12-05T11:57:36,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/1ebf90d4ca4149d4a349fa9eb5a265d3 is 50, key is test_row_0/C:col10/1733399854627/Put/seqid=0 2024-12-05T11:57:36,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742123_1299 (size=12301) 2024-12-05T11:57:36,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T11:57:36,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:36,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:36,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:36,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:36,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T11:57:36,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T11:57:36,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:36,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:36,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:36,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:36,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:36,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399916407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:36,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399916407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:36,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399916407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:36,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T11:57:36,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:36,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:36,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:36,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:36,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T11:57:36,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/1ebf90d4ca4149d4a349fa9eb5a265d3 2024-12-05T11:57:36,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/69dbbf2fc5b64d9e83893c4e91cc34c2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/69dbbf2fc5b64d9e83893c4e91cc34c2 2024-12-05T11:57:36,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/69dbbf2fc5b64d9e83893c4e91cc34c2, entries=200, sequenceid=357, filesize=14.4 K 2024-12-05T11:57:36,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/6aef96b3140d4fc8a3ae01ba0a79cbc7 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/6aef96b3140d4fc8a3ae01ba0a79cbc7 2024-12-05T11:57:36,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/6aef96b3140d4fc8a3ae01ba0a79cbc7, entries=150, sequenceid=357, filesize=12.0 K 2024-12-05T11:57:36,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/1ebf90d4ca4149d4a349fa9eb5a265d3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/1ebf90d4ca4149d4a349fa9eb5a265d3 2024-12-05T11:57:36,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/1ebf90d4ca4149d4a349fa9eb5a265d3, entries=150, sequenceid=357, filesize=12.0 K 2024-12-05T11:57:36,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for d5e0da544abea90f443151cdd8418bf7 in 1246ms, sequenceid=357, compaction requested=true 2024-12-05T11:57:36,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:36,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:57:36,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:36,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:57:36,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:36,505 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:36,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:57:36,505 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:36,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:36,506 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:36,506 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/B is initiating minor compaction (all files) 2024-12-05T11:57:36,506 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/B in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:36,506 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/55485a13f532415c9c209690846736f2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/0dc9cfcc594349a489075d983006cacb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/6aef96b3140d4fc8a3ae01ba0a79cbc7] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=36.8 K 2024-12-05T11:57:36,506 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:36,506 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/A is initiating minor compaction (all files) 2024-12-05T11:57:36,506 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/A in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:36,506 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/f7500c7d397143799118234ca604a546, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/6c687c2670fa48de81330114222a4105, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/69dbbf2fc5b64d9e83893c4e91cc34c2] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=39.2 K 2024-12-05T11:57:36,507 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7500c7d397143799118234ca604a546, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733399852313 2024-12-05T11:57:36,507 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 55485a13f532415c9c209690846736f2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733399852313 2024-12-05T11:57:36,507 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c687c2670fa48de81330114222a4105, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733399853460 2024-12-05T11:57:36,507 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc9cfcc594349a489075d983006cacb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733399853460 2024-12-05T11:57:36,507 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69dbbf2fc5b64d9e83893c4e91cc34c2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733399854625 2024-12-05T11:57:36,508 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aef96b3140d4fc8a3ae01ba0a79cbc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733399854627 2024-12-05T11:57:36,515 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:36,515 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/da9028e6d7244ba78e7dabbcac9107c1 is 50, key is test_row_0/B:col10/1733399854627/Put/seqid=0 2024-12-05T11:57:36,516 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:36,517 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/d30d68e3829043959cc90e58eb801cd8 is 50, key is test_row_0/A:col10/1733399854627/Put/seqid=0 2024-12-05T11:57:36,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742124_1300 (size=13153) 2024-12-05T11:57:36,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742125_1301 (size=13153) 2024-12-05T11:57:36,547 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/da9028e6d7244ba78e7dabbcac9107c1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/da9028e6d7244ba78e7dabbcac9107c1 2024-12-05T11:57:36,552 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/B of d5e0da544abea90f443151cdd8418bf7 into da9028e6d7244ba78e7dabbcac9107c1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:36,552 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:36,552 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/B, priority=13, startTime=1733399856505; duration=0sec 2024-12-05T11:57:36,552 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:36,552 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:B 2024-12-05T11:57:36,552 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:36,553 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:36,553 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/C is initiating minor compaction (all files) 2024-12-05T11:57:36,553 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/C in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:36,553 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ee887bb26aab4dfc9a4aba860a3085eb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/902cb488d9014f7d9fe7001b0601116b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/1ebf90d4ca4149d4a349fa9eb5a265d3] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=36.8 K 2024-12-05T11:57:36,553 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ee887bb26aab4dfc9a4aba860a3085eb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733399852313 2024-12-05T11:57:36,554 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 902cb488d9014f7d9fe7001b0601116b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733399853460 2024-12-05T11:57:36,554 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ebf90d4ca4149d4a349fa9eb5a265d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733399854627 2024-12-05T11:57:36,560 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:36,560 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/c4109ff3d5cb4d4e92264787c94680c8 is 50, key is test_row_0/C:col10/1733399854627/Put/seqid=0 2024-12-05T11:57:36,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742126_1302 (size=13153) 2024-12-05T11:57:36,569 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/c4109ff3d5cb4d4e92264787c94680c8 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/c4109ff3d5cb4d4e92264787c94680c8 2024-12-05T11:57:36,573 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/C of d5e0da544abea90f443151cdd8418bf7 into c4109ff3d5cb4d4e92264787c94680c8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:36,573 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:36,573 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/C, priority=13, startTime=1733399856505; duration=0sec 2024-12-05T11:57:36,574 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:36,574 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:C 2024-12-05T11:57:36,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T11:57:36,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:36,623 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-05T11:57:36,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/5f694f95478c41eaad79e1490d3b037b is 50, key is test_row_0/A:col10/1733399855264/Put/seqid=0 2024-12-05T11:57:36,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742127_1303 (size=12301) 2024-12-05T11:57:36,947 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/d30d68e3829043959cc90e58eb801cd8 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d30d68e3829043959cc90e58eb801cd8 2024-12-05T11:57:36,951 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/A of d5e0da544abea90f443151cdd8418bf7 into d30d68e3829043959cc90e58eb801cd8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:36,951 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:36,951 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/A, priority=13, startTime=1733399856505; duration=0sec 2024-12-05T11:57:36,952 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:36,952 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:A 2024-12-05T11:57:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T11:57:37,035 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/5f694f95478c41eaad79e1490d3b037b 2024-12-05T11:57:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/a0b335f0d5784d30be85f2050af0c12d is 50, key is test_row_0/B:col10/1733399855264/Put/seqid=0 2024-12-05T11:57:37,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742128_1304 (size=12301) 2024-12-05T11:57:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:37,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:37,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:37,453 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/a0b335f0d5784d30be85f2050af0c12d 2024-12-05T11:57:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399917450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:37,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399917452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:37,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399917457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:37,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/56a1bc3276aa4660aaae11d7d0a51c61 is 50, key is test_row_0/C:col10/1733399855264/Put/seqid=0 2024-12-05T11:57:37,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742129_1305 (size=12301) 2024-12-05T11:57:37,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399917558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:37,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:37,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399917558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:37,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:37,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399917567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:37,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:37,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399917768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:37,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399917768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:37,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399917777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:37,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:37,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:37,869 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/56a1bc3276aa4660aaae11d7d0a51c61 2024-12-05T11:57:37,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/5f694f95478c41eaad79e1490d3b037b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/5f694f95478c41eaad79e1490d3b037b 2024-12-05T11:57:37,884 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/5f694f95478c41eaad79e1490d3b037b, entries=150, sequenceid=369, filesize=12.0 K 2024-12-05T11:57:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/a0b335f0d5784d30be85f2050af0c12d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a0b335f0d5784d30be85f2050af0c12d 2024-12-05T11:57:37,891 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a0b335f0d5784d30be85f2050af0c12d, entries=150, sequenceid=369, filesize=12.0 K 2024-12-05T11:57:37,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/56a1bc3276aa4660aaae11d7d0a51c61 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/56a1bc3276aa4660aaae11d7d0a51c61 2024-12-05T11:57:37,900 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/56a1bc3276aa4660aaae11d7d0a51c61, entries=150, sequenceid=369, filesize=12.0 K 2024-12-05T11:57:37,902 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for d5e0da544abea90f443151cdd8418bf7 in 1279ms, sequenceid=369, compaction requested=false 2024-12-05T11:57:37,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:37,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:37,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-05T11:57:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-05T11:57:37,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-05T11:57:37,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0430 sec 2024-12-05T11:57:37,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.0480 sec 2024-12-05T11:57:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T11:57:38,008 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:38,009 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-05T11:57:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T11:57:38,010 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:38,011 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:38,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:38,034 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T11:57:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:38,080 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:38,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-12-05T11:57:38,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/d1fb4782c6f4437594eb2bb7cf674856 is 50, key is test_row_0/A:col10/1733399857449/Put/seqid=0 2024-12-05T11:57:38,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742130_1306 (size=14741) 2024-12-05T11:57:38,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399918086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:38,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:38,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399918088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:38,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399918089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T11:57:38,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T11:57:38,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:38,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:38,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:38,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:38,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399918198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399918198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:38,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T11:57:38,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:38,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:38,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:38,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T11:57:38,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399918408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:38,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399918408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:38,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T11:57:38,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:38,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:38,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:38,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/d1fb4782c6f4437594eb2bb7cf674856 2024-12-05T11:57:38,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/b248a604cbb04cd69ff200817cdf08bf is 50, key is test_row_0/B:col10/1733399857449/Put/seqid=0 2024-12-05T11:57:38,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742131_1307 (size=12301) 2024-12-05T11:57:38,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733399918597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:38,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T11:57:38,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:38,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:38,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:38,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T11:57:38,645 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-12-05T11:57:38,645 INFO [Thread-1067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1067. 2024-12-05T11:57:38,645 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:38,645 DEBUG [Thread-1067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:38,645 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:38,645 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:38,645 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:38,645 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:38,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:38,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:38,646 INFO [Thread-1065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1065. 2024-12-05T11:57:38,646 DEBUG [Thread-1065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:38,646 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:38,646 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:38,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:38,647 INFO [Thread-1063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1063. 2024-12-05T11:57:38,647 DEBUG [Thread-1063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:38,647 DEBUG [Thread-1063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:38,647 DEBUG [Thread-1063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:38,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:38,647 INFO [Thread-1069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1069. 2024-12-05T11:57:38,647 DEBUG [Thread-1069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:38,647 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:38,647 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:38,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:38,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:38,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399918718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399918718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:38,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T11:57:38,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:38,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/b248a604cbb04cd69ff200817cdf08bf 2024-12-05T11:57:38,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/013f813ce7eb45ec830025e55869cf04 is 50, key is test_row_0/C:col10/1733399857449/Put/seqid=0 2024-12-05T11:57:38,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742132_1308 (size=12301) 2024-12-05T11:57:38,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T11:57:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:38,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:39,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T11:57:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:39,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:39,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:39,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:39,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:39,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53438 deadline: 1733399919128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:39,130 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in d5e0da544abea90f443151cdd8418bf7 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T11:57:39,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53412 deadline: 1733399919158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:39,160 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in d5e0da544abea90f443151cdd8418bf7 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:39,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:39,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:39,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:39,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:39,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53374 deadline: 1733399919228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53434 deadline: 1733399919229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:57:39,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T11:57:39,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:39,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. as already flushing 2024-12-05T11:57:39,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:39,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:39,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:39,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/013f813ce7eb45ec830025e55869cf04 2024-12-05T11:57:39,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/d1fb4782c6f4437594eb2bb7cf674856 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d1fb4782c6f4437594eb2bb7cf674856 2024-12-05T11:57:39,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d1fb4782c6f4437594eb2bb7cf674856, entries=200, sequenceid=399, filesize=14.4 K 2024-12-05T11:57:39,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/b248a604cbb04cd69ff200817cdf08bf as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b248a604cbb04cd69ff200817cdf08bf 2024-12-05T11:57:39,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b248a604cbb04cd69ff200817cdf08bf, entries=150, sequenceid=399, filesize=12.0 K 2024-12-05T11:57:39,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/013f813ce7eb45ec830025e55869cf04 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/013f813ce7eb45ec830025e55869cf04 2024-12-05T11:57:39,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/013f813ce7eb45ec830025e55869cf04, entries=150, sequenceid=399, filesize=12.0 K 2024-12-05T11:57:39,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for d5e0da544abea90f443151cdd8418bf7 in 1263ms, sequenceid=399, compaction requested=true 2024-12-05T11:57:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:57:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:39,343 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:57:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:39,343 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5e0da544abea90f443151cdd8418bf7:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:57:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:39,344 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:39,344 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:39,344 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/B is initiating minor compaction (all files) 2024-12-05T11:57:39,344 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/A is initiating minor compaction (all files) 2024-12-05T11:57:39,344 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/A in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:39,344 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/B in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:39,344 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/da9028e6d7244ba78e7dabbcac9107c1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a0b335f0d5784d30be85f2050af0c12d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b248a604cbb04cd69ff200817cdf08bf] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=36.9 K 2024-12-05T11:57:39,344 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d30d68e3829043959cc90e58eb801cd8, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/5f694f95478c41eaad79e1490d3b037b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d1fb4782c6f4437594eb2bb7cf674856] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=39.3 K 2024-12-05T11:57:39,345 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d30d68e3829043959cc90e58eb801cd8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733399854627 2024-12-05T11:57:39,345 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting da9028e6d7244ba78e7dabbcac9107c1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733399854627 2024-12-05T11:57:39,345 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0b335f0d5784d30be85f2050af0c12d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733399855261 2024-12-05T11:57:39,345 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f694f95478c41eaad79e1490d3b037b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733399855261 2024-12-05T11:57:39,345 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b248a604cbb04cd69ff200817cdf08bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733399857449 2024-12-05T11:57:39,345 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d1fb4782c6f4437594eb2bb7cf674856, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733399857449 2024-12-05T11:57:39,351 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:39,351 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:39,352 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/38d8644a22bc4e90b6bc118c0a8cd034 is 50, key is test_row_0/A:col10/1733399857449/Put/seqid=0 2024-12-05T11:57:39,352 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/2fe36fd4494f40c4a294e254928d8e5c is 50, key is test_row_0/B:col10/1733399857449/Put/seqid=0 2024-12-05T11:57:39,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742134_1310 (size=13255) 2024-12-05T11:57:39,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742133_1309 (size=13255) 2024-12-05T11:57:39,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T11:57:39,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:39,396 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-05T11:57:39,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/77d89a6b049c4777b7ba2a66c72230ec is 50, key is test_row_0/A:col10/1733399858082/Put/seqid=0 2024-12-05T11:57:39,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742135_1311 (size=12301) 2024-12-05T11:57:39,634 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-12-05T11:57:39,634 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:39,634 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:39,634 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:39,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:39,759 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/38d8644a22bc4e90b6bc118c0a8cd034 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/38d8644a22bc4e90b6bc118c0a8cd034 2024-12-05T11:57:39,759 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/2fe36fd4494f40c4a294e254928d8e5c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/2fe36fd4494f40c4a294e254928d8e5c 2024-12-05T11:57:39,763 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/A of d5e0da544abea90f443151cdd8418bf7 into 38d8644a22bc4e90b6bc118c0a8cd034(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:39,763 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/B of d5e0da544abea90f443151cdd8418bf7 into 2fe36fd4494f40c4a294e254928d8e5c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:39,763 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:39,763 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:39,763 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/A, priority=13, startTime=1733399859343; duration=0sec 2024-12-05T11:57:39,763 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/B, priority=13, startTime=1733399859343; duration=0sec 2024-12-05T11:57:39,763 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:39,763 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:A 2024-12-05T11:57:39,763 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:39,763 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:B 2024-12-05T11:57:39,763 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:39,764 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:39,764 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): d5e0da544abea90f443151cdd8418bf7/C is initiating minor compaction (all files) 2024-12-05T11:57:39,764 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5e0da544abea90f443151cdd8418bf7/C in TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:39,764 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/c4109ff3d5cb4d4e92264787c94680c8, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/56a1bc3276aa4660aaae11d7d0a51c61, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/013f813ce7eb45ec830025e55869cf04] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp, totalSize=36.9 K 2024-12-05T11:57:39,764 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c4109ff3d5cb4d4e92264787c94680c8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733399854627 2024-12-05T11:57:39,765 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a1bc3276aa4660aaae11d7d0a51c61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733399855261 2024-12-05T11:57:39,765 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 013f813ce7eb45ec830025e55869cf04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733399857449 2024-12-05T11:57:39,771 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5e0da544abea90f443151cdd8418bf7#C#compaction#270 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:39,771 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/f581201e37a440dba2bd391cb4e74122 is 50, key is test_row_0/C:col10/1733399857449/Put/seqid=0 2024-12-05T11:57:39,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742136_1312 (size=13255) 2024-12-05T11:57:39,803 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/77d89a6b049c4777b7ba2a66c72230ec 2024-12-05T11:57:39,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/69aef949db9646ce82f2c780761fefda is 50, key is test_row_0/B:col10/1733399858082/Put/seqid=0 2024-12-05T11:57:39,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742137_1313 (size=12301) 2024-12-05T11:57:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T11:57:40,187 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/f581201e37a440dba2bd391cb4e74122 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/f581201e37a440dba2bd391cb4e74122 2024-12-05T11:57:40,194 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5e0da544abea90f443151cdd8418bf7/C of d5e0da544abea90f443151cdd8418bf7 into f581201e37a440dba2bd391cb4e74122(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:40,194 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:40,194 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., storeName=d5e0da544abea90f443151cdd8418bf7/C, priority=13, startTime=1733399859343; duration=0sec 2024-12-05T11:57:40,194 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:40,194 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5e0da544abea90f443151cdd8418bf7:C 2024-12-05T11:57:40,217 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/69aef949db9646ce82f2c780761fefda 2024-12-05T11:57:40,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/e66203f356584affa1cb437fd6752e06 is 50, key is test_row_0/C:col10/1733399858082/Put/seqid=0 2024-12-05T11:57:40,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742138_1314 (size=12301) 2024-12-05T11:57:40,240 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-12-05T11:57:40,240 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:40,240 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:40,240 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:40,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:40,250 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-12-05T11:57:40,250 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:40,250 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:40,250 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:40,250 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:40,631 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/e66203f356584affa1cb437fd6752e06 2024-12-05T11:57:40,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/77d89a6b049c4777b7ba2a66c72230ec as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/77d89a6b049c4777b7ba2a66c72230ec 2024-12-05T11:57:40,648 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/77d89a6b049c4777b7ba2a66c72230ec, entries=150, sequenceid=406, filesize=12.0 K 2024-12-05T11:57:40,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/69aef949db9646ce82f2c780761fefda as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/69aef949db9646ce82f2c780761fefda 2024-12-05T11:57:40,653 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/69aef949db9646ce82f2c780761fefda, entries=150, sequenceid=406, filesize=12.0 K 2024-12-05T11:57:40,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/e66203f356584affa1cb437fd6752e06 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/e66203f356584affa1cb437fd6752e06 2024-12-05T11:57:40,657 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/e66203f356584affa1cb437fd6752e06, entries=150, sequenceid=406, filesize=12.0 K 2024-12-05T11:57:40,658 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=20.13 KB/20610 for d5e0da544abea90f443151cdd8418bf7 in 1262ms, sequenceid=406, compaction requested=false 2024-12-05T11:57:40,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for d5e0da544abea90f443151cdd8418bf7: 2024-12-05T11:57:40,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:40,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-05T11:57:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-05T11:57:40,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-05T11:57:40,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6490 sec 2024-12-05T11:57:40,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.6530 sec 2024-12-05T11:57:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T11:57:42,179 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:49,199 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-12-05T11:57:49,199 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:49,199 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:49,200 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:49,200 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:49,219 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-12-05T11:57:49,219 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:57:49,219 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:49,219 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:49,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5520 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5383 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5591 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5514 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5376 2024-12-05T11:57:49,220 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T11:57:49,220 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:57:49,220 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:57:49,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:49,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:49,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:57:49,221 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T11:57:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T11:57:49,225 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399869225"}]},"ts":"1733399869225"} 2024-12-05T11:57:49,226 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T11:57:49,226 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T11:57:49,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:57:49,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5e0da544abea90f443151cdd8418bf7, UNASSIGN}] 2024-12-05T11:57:49,229 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5e0da544abea90f443151cdd8418bf7, UNASSIGN 2024-12-05T11:57:49,229 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=d5e0da544abea90f443151cdd8418bf7, regionState=CLOSING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:57:49,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5e0da544abea90f443151cdd8418bf7, UNASSIGN because future has completed 2024-12-05T11:57:49,231 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:57:49,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:57:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T11:57:49,384 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:49,384 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:57:49,384 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing d5e0da544abea90f443151cdd8418bf7, disabling compactions & flushes 2024-12-05T11:57:49,384 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:49,384 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:49,384 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. after waiting 0 ms 2024-12-05T11:57:49,384 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:49,384 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing d5e0da544abea90f443151cdd8418bf7 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-05T11:57:49,388 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/bdf8f82e62de43c08a98812649256973 is 50, key is test_row_0/A:col10/1733399869198/Put/seqid=0 2024-12-05T11:57:49,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742139_1315 (size=12301) 2024-12-05T11:57:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T11:57:49,792 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/bdf8f82e62de43c08a98812649256973 2024-12-05T11:57:49,800 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/4349bcbbe9994b83bb0f4e6298a6c6da is 50, key is test_row_0/B:col10/1733399869198/Put/seqid=0 2024-12-05T11:57:49,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742140_1316 (size=12301) 2024-12-05T11:57:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T11:57:50,205 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/4349bcbbe9994b83bb0f4e6298a6c6da 2024-12-05T11:57:50,213 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/9cea0bb3c9044fbe9cd43daa935746f4 is 50, key is test_row_0/C:col10/1733399869198/Put/seqid=0 2024-12-05T11:57:50,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742141_1317 (size=12301) 2024-12-05T11:57:50,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T11:57:50,618 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/9cea0bb3c9044fbe9cd43daa935746f4 2024-12-05T11:57:50,622 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/A/bdf8f82e62de43c08a98812649256973 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/bdf8f82e62de43c08a98812649256973 2024-12-05T11:57:50,626 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/bdf8f82e62de43c08a98812649256973, entries=150, sequenceid=417, filesize=12.0 K 2024-12-05T11:57:50,627 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/B/4349bcbbe9994b83bb0f4e6298a6c6da as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/4349bcbbe9994b83bb0f4e6298a6c6da 2024-12-05T11:57:50,631 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/4349bcbbe9994b83bb0f4e6298a6c6da, entries=150, sequenceid=417, filesize=12.0 K 2024-12-05T11:57:50,632 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/.tmp/C/9cea0bb3c9044fbe9cd43daa935746f4 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/9cea0bb3c9044fbe9cd43daa935746f4 2024-12-05T11:57:50,636 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/9cea0bb3c9044fbe9cd43daa935746f4, entries=150, sequenceid=417, filesize=12.0 K 2024-12-05T11:57:50,636 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d5e0da544abea90f443151cdd8418bf7 in 1252ms, sequenceid=417, compaction requested=true 2024-12-05T11:57:50,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/f0fad698219b4b1e8e4b99a37be8362a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/9097346942574861b1f0e19861e53c09, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/2d738e02d58e4b6baf41a033e71f9f6d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8c6518978a464292b687c52290bf2161, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8b8aa3c7aa1443a19597963ca5d83072, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/3d11b265b6314bee86b233a1d6e26307, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/1e9ad491c0b343a8a03715e3d67c1238, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8c48cd5f80994d8d9d3601005146fdd4, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/b9b15cdea9d3409a957d221145233d0d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/08331da841014a019ac021976c65f409, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/dd61cbf13f644c22b8f30e6c1a9e9d21, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/09177793fae4475a879d45321177e95c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/c84b10e4401b4de4a4c31a0a2e55bbb7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d538e8fe54c44f31880233da16160c61, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/83030c555b514a28963e5a674859e0b3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/afe7d523a8324ad4bb42a88d024ae2a9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/cf98f3ac0f1f42529628fc477e220361, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/c399a258a23d4f5f9b3e545009836826, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/120a84ba7d1949998bf9cc7250c92d85, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/e31c23f9fc604afea3e0bfd91cd66968, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/835439ef76aa47259a8fc28951290b38, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d3be594ccc0e4937b5272d84ebf24353, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/f7500c7d397143799118234ca604a546, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/6c687c2670fa48de81330114222a4105, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/69dbbf2fc5b64d9e83893c4e91cc34c2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d30d68e3829043959cc90e58eb801cd8, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/5f694f95478c41eaad79e1490d3b037b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d1fb4782c6f4437594eb2bb7cf674856] to archive 2024-12-05T11:57:50,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:57:50,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/f0fad698219b4b1e8e4b99a37be8362a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/f0fad698219b4b1e8e4b99a37be8362a 2024-12-05T11:57:50,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/9097346942574861b1f0e19861e53c09 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/9097346942574861b1f0e19861e53c09 2024-12-05T11:57:50,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/2d738e02d58e4b6baf41a033e71f9f6d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/2d738e02d58e4b6baf41a033e71f9f6d 2024-12-05T11:57:50,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8c6518978a464292b687c52290bf2161 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8c6518978a464292b687c52290bf2161 2024-12-05T11:57:50,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8b8aa3c7aa1443a19597963ca5d83072 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8b8aa3c7aa1443a19597963ca5d83072 2024-12-05T11:57:50,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/3d11b265b6314bee86b233a1d6e26307 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/3d11b265b6314bee86b233a1d6e26307 2024-12-05T11:57:50,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/1e9ad491c0b343a8a03715e3d67c1238 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/1e9ad491c0b343a8a03715e3d67c1238 2024-12-05T11:57:50,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8c48cd5f80994d8d9d3601005146fdd4 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/8c48cd5f80994d8d9d3601005146fdd4 2024-12-05T11:57:50,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/b9b15cdea9d3409a957d221145233d0d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/b9b15cdea9d3409a957d221145233d0d 2024-12-05T11:57:50,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/08331da841014a019ac021976c65f409 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/08331da841014a019ac021976c65f409 2024-12-05T11:57:50,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/dd61cbf13f644c22b8f30e6c1a9e9d21 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/dd61cbf13f644c22b8f30e6c1a9e9d21 2024-12-05T11:57:50,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/09177793fae4475a879d45321177e95c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/09177793fae4475a879d45321177e95c 2024-12-05T11:57:50,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/c84b10e4401b4de4a4c31a0a2e55bbb7 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/c84b10e4401b4de4a4c31a0a2e55bbb7 2024-12-05T11:57:50,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d538e8fe54c44f31880233da16160c61 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d538e8fe54c44f31880233da16160c61 2024-12-05T11:57:50,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/83030c555b514a28963e5a674859e0b3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/83030c555b514a28963e5a674859e0b3 2024-12-05T11:57:50,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/afe7d523a8324ad4bb42a88d024ae2a9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/afe7d523a8324ad4bb42a88d024ae2a9 2024-12-05T11:57:50,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/cf98f3ac0f1f42529628fc477e220361 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/cf98f3ac0f1f42529628fc477e220361 2024-12-05T11:57:50,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/c399a258a23d4f5f9b3e545009836826 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/c399a258a23d4f5f9b3e545009836826 2024-12-05T11:57:50,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/120a84ba7d1949998bf9cc7250c92d85 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/120a84ba7d1949998bf9cc7250c92d85 2024-12-05T11:57:50,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/e31c23f9fc604afea3e0bfd91cd66968 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/e31c23f9fc604afea3e0bfd91cd66968 2024-12-05T11:57:50,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/835439ef76aa47259a8fc28951290b38 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/835439ef76aa47259a8fc28951290b38 2024-12-05T11:57:50,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d3be594ccc0e4937b5272d84ebf24353 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d3be594ccc0e4937b5272d84ebf24353 2024-12-05T11:57:50,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/f7500c7d397143799118234ca604a546 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/f7500c7d397143799118234ca604a546 2024-12-05T11:57:50,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/6c687c2670fa48de81330114222a4105 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/6c687c2670fa48de81330114222a4105 2024-12-05T11:57:50,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/69dbbf2fc5b64d9e83893c4e91cc34c2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/69dbbf2fc5b64d9e83893c4e91cc34c2 2024-12-05T11:57:50,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d30d68e3829043959cc90e58eb801cd8 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d30d68e3829043959cc90e58eb801cd8 2024-12-05T11:57:50,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/5f694f95478c41eaad79e1490d3b037b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/5f694f95478c41eaad79e1490d3b037b 2024-12-05T11:57:50,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d1fb4782c6f4437594eb2bb7cf674856 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/d1fb4782c6f4437594eb2bb7cf674856 2024-12-05T11:57:50,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a06dc8a7b4e447e68951a5851b42c295, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b3e0b276933749b7ba232121787b8cb8, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a62831be35cb4da996b3a2aa1c0a5796, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/27b67c0d32b2414db56e203de95fdad0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b2a70888a9a748dcab9e594a090ddcd6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/11c714302fd0490e881bfaa70d0535c6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f6f8d2df02694b2db1c44d6f03cb1596, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/6130d88122cd4c96b07e3b3a7dc9abab, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/d7acf4b159ff4e0888c35af020794ec0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/d88700818e4249e9921f147a48b6c4f9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/eaf8537cc75b4cd88ce447d4d04d1975, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f9847076e71e47b6a01a51ae44e7e408, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/1d5f5fc6d1ee494a872f8c63f5c2cdb3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/4320a7a9000842d0b9656e1697beee9c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/5bdec0b3048a4812881720a76ad33550, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/78696ff61e3f4f0daf2b78262d37403d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/1267570cad91423088125a387bac9856, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/c1827ebe565147b58b4bfa6e3a5c0362, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b948b27193f442d7b45bbd53edea40ab, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f4d423f1d90c4b4081534bc8756f0f0c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/fb45c341671a4cebb3298258b1bcfa6d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/55485a13f532415c9c209690846736f2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/964563b9c683496d8b1770b0f0a8f947, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/0dc9cfcc594349a489075d983006cacb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/da9028e6d7244ba78e7dabbcac9107c1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/6aef96b3140d4fc8a3ae01ba0a79cbc7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a0b335f0d5784d30be85f2050af0c12d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b248a604cbb04cd69ff200817cdf08bf] to archive 2024-12-05T11:57:50,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:57:50,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a06dc8a7b4e447e68951a5851b42c295 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a06dc8a7b4e447e68951a5851b42c295 2024-12-05T11:57:50,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b3e0b276933749b7ba232121787b8cb8 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b3e0b276933749b7ba232121787b8cb8 2024-12-05T11:57:50,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a62831be35cb4da996b3a2aa1c0a5796 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a62831be35cb4da996b3a2aa1c0a5796 2024-12-05T11:57:50,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/27b67c0d32b2414db56e203de95fdad0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/27b67c0d32b2414db56e203de95fdad0 2024-12-05T11:57:50,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b2a70888a9a748dcab9e594a090ddcd6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b2a70888a9a748dcab9e594a090ddcd6 2024-12-05T11:57:50,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/11c714302fd0490e881bfaa70d0535c6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/11c714302fd0490e881bfaa70d0535c6 2024-12-05T11:57:50,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f6f8d2df02694b2db1c44d6f03cb1596 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f6f8d2df02694b2db1c44d6f03cb1596 2024-12-05T11:57:50,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/6130d88122cd4c96b07e3b3a7dc9abab to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/6130d88122cd4c96b07e3b3a7dc9abab 2024-12-05T11:57:50,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/d7acf4b159ff4e0888c35af020794ec0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/d7acf4b159ff4e0888c35af020794ec0 2024-12-05T11:57:50,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/d88700818e4249e9921f147a48b6c4f9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/d88700818e4249e9921f147a48b6c4f9 2024-12-05T11:57:50,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/eaf8537cc75b4cd88ce447d4d04d1975 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/eaf8537cc75b4cd88ce447d4d04d1975 2024-12-05T11:57:50,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f9847076e71e47b6a01a51ae44e7e408 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f9847076e71e47b6a01a51ae44e7e408 2024-12-05T11:57:50,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/1d5f5fc6d1ee494a872f8c63f5c2cdb3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/1d5f5fc6d1ee494a872f8c63f5c2cdb3 2024-12-05T11:57:50,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/4320a7a9000842d0b9656e1697beee9c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/4320a7a9000842d0b9656e1697beee9c 2024-12-05T11:57:50,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/5bdec0b3048a4812881720a76ad33550 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/5bdec0b3048a4812881720a76ad33550 2024-12-05T11:57:50,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/78696ff61e3f4f0daf2b78262d37403d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/78696ff61e3f4f0daf2b78262d37403d 2024-12-05T11:57:50,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/1267570cad91423088125a387bac9856 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/1267570cad91423088125a387bac9856 2024-12-05T11:57:50,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/c1827ebe565147b58b4bfa6e3a5c0362 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/c1827ebe565147b58b4bfa6e3a5c0362 2024-12-05T11:57:50,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b948b27193f442d7b45bbd53edea40ab to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b948b27193f442d7b45bbd53edea40ab 2024-12-05T11:57:50,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f4d423f1d90c4b4081534bc8756f0f0c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/f4d423f1d90c4b4081534bc8756f0f0c 2024-12-05T11:57:50,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/fb45c341671a4cebb3298258b1bcfa6d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/fb45c341671a4cebb3298258b1bcfa6d 2024-12-05T11:57:50,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/55485a13f532415c9c209690846736f2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/55485a13f532415c9c209690846736f2 2024-12-05T11:57:50,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/964563b9c683496d8b1770b0f0a8f947 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/964563b9c683496d8b1770b0f0a8f947 2024-12-05T11:57:50,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/0dc9cfcc594349a489075d983006cacb to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/0dc9cfcc594349a489075d983006cacb 2024-12-05T11:57:50,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/da9028e6d7244ba78e7dabbcac9107c1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/da9028e6d7244ba78e7dabbcac9107c1 2024-12-05T11:57:50,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/6aef96b3140d4fc8a3ae01ba0a79cbc7 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/6aef96b3140d4fc8a3ae01ba0a79cbc7 2024-12-05T11:57:50,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a0b335f0d5784d30be85f2050af0c12d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/a0b335f0d5784d30be85f2050af0c12d 2024-12-05T11:57:50,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b248a604cbb04cd69ff200817cdf08bf to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/b248a604cbb04cd69ff200817cdf08bf 2024-12-05T11:57:50,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/81f8a4fa3e7d4c3f9f7d0700d831261e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ba61f1f28d144945bf176dbe848c030c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/8625874616bb4c3798dd7cc8ef444646, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ca4c021c78704b089146e246c2fa9b9e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/a0d2d738064540569e16bd771d208cbe, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4ab95760172c463eb7845a77cb90caf1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/24e0222dd7db42eb972811b658bf35b0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/344d33e57b6f4564b68cb943800e6e2f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ef72ab4410ba49fb82bbdf00cb134f5c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4d0bb05b8f974cc2880551163d02c668, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/cf60c2d46d9f45bc90c0e209b5c7dc6f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/1bd74d0b6d444c05831d5afd7b78bd21, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/6f4998b931d843a59cfabe43bec749e9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4e76f8e68e8f452e806cd1adeebd03a2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ce9fe20c27414eb5a77ba4e4f143d225, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/353a71cf60b64e89a2947fb9caccfcf9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/18081e3726c34fb59bdcca91d8b7d252, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/28bd33a308c144459970a1128f37a0cf, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/dbac7ce1468c4ce6b6dd82111c188f5b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/585d8e93e9d14c8c8c69cf6685b2dbd7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/078f007f842f40c7b8178d38aca441ca, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ee887bb26aab4dfc9a4aba860a3085eb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ca76f836db88491fbe6a3d186a6b1d6b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/902cb488d9014f7d9fe7001b0601116b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/c4109ff3d5cb4d4e92264787c94680c8, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/1ebf90d4ca4149d4a349fa9eb5a265d3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/56a1bc3276aa4660aaae11d7d0a51c61, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/013f813ce7eb45ec830025e55869cf04] to archive 2024-12-05T11:57:50,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:57:50,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/81f8a4fa3e7d4c3f9f7d0700d831261e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/81f8a4fa3e7d4c3f9f7d0700d831261e 2024-12-05T11:57:50,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ba61f1f28d144945bf176dbe848c030c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ba61f1f28d144945bf176dbe848c030c 2024-12-05T11:57:50,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/8625874616bb4c3798dd7cc8ef444646 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/8625874616bb4c3798dd7cc8ef444646 2024-12-05T11:57:50,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ca4c021c78704b089146e246c2fa9b9e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ca4c021c78704b089146e246c2fa9b9e 2024-12-05T11:57:50,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/a0d2d738064540569e16bd771d208cbe to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/a0d2d738064540569e16bd771d208cbe 2024-12-05T11:57:50,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4ab95760172c463eb7845a77cb90caf1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4ab95760172c463eb7845a77cb90caf1 2024-12-05T11:57:50,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/24e0222dd7db42eb972811b658bf35b0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/24e0222dd7db42eb972811b658bf35b0 2024-12-05T11:57:50,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/344d33e57b6f4564b68cb943800e6e2f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/344d33e57b6f4564b68cb943800e6e2f 2024-12-05T11:57:50,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ef72ab4410ba49fb82bbdf00cb134f5c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ef72ab4410ba49fb82bbdf00cb134f5c 2024-12-05T11:57:50,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4d0bb05b8f974cc2880551163d02c668 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4d0bb05b8f974cc2880551163d02c668 2024-12-05T11:57:50,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/cf60c2d46d9f45bc90c0e209b5c7dc6f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/cf60c2d46d9f45bc90c0e209b5c7dc6f 2024-12-05T11:57:50,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/1bd74d0b6d444c05831d5afd7b78bd21 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/1bd74d0b6d444c05831d5afd7b78bd21 2024-12-05T11:57:50,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/6f4998b931d843a59cfabe43bec749e9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/6f4998b931d843a59cfabe43bec749e9 2024-12-05T11:57:50,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4e76f8e68e8f452e806cd1adeebd03a2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/4e76f8e68e8f452e806cd1adeebd03a2 2024-12-05T11:57:50,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ce9fe20c27414eb5a77ba4e4f143d225 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ce9fe20c27414eb5a77ba4e4f143d225 2024-12-05T11:57:50,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/353a71cf60b64e89a2947fb9caccfcf9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/353a71cf60b64e89a2947fb9caccfcf9 2024-12-05T11:57:50,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/18081e3726c34fb59bdcca91d8b7d252 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/18081e3726c34fb59bdcca91d8b7d252 2024-12-05T11:57:50,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/28bd33a308c144459970a1128f37a0cf to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/28bd33a308c144459970a1128f37a0cf 2024-12-05T11:57:50,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/dbac7ce1468c4ce6b6dd82111c188f5b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/dbac7ce1468c4ce6b6dd82111c188f5b 2024-12-05T11:57:50,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/585d8e93e9d14c8c8c69cf6685b2dbd7 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/585d8e93e9d14c8c8c69cf6685b2dbd7 2024-12-05T11:57:50,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/078f007f842f40c7b8178d38aca441ca to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/078f007f842f40c7b8178d38aca441ca 2024-12-05T11:57:50,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ee887bb26aab4dfc9a4aba860a3085eb to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ee887bb26aab4dfc9a4aba860a3085eb 2024-12-05T11:57:50,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ca76f836db88491fbe6a3d186a6b1d6b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/ca76f836db88491fbe6a3d186a6b1d6b 2024-12-05T11:57:50,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/902cb488d9014f7d9fe7001b0601116b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/902cb488d9014f7d9fe7001b0601116b 2024-12-05T11:57:50,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/c4109ff3d5cb4d4e92264787c94680c8 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/c4109ff3d5cb4d4e92264787c94680c8 2024-12-05T11:57:50,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/1ebf90d4ca4149d4a349fa9eb5a265d3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/1ebf90d4ca4149d4a349fa9eb5a265d3 2024-12-05T11:57:50,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/56a1bc3276aa4660aaae11d7d0a51c61 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/56a1bc3276aa4660aaae11d7d0a51c61 2024-12-05T11:57:50,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/013f813ce7eb45ec830025e55869cf04 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/013f813ce7eb45ec830025e55869cf04 2024-12-05T11:57:50,759 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/recovered.edits/420.seqid, newMaxSeqId=420, maxSeqId=1 2024-12-05T11:57:50,760 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. 2024-12-05T11:57:50,760 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for d5e0da544abea90f443151cdd8418bf7: Waiting for close lock at 1733399869384Running coprocessor pre-close hooks at 1733399869384Disabling compacts and flushes for region at 1733399869384Disabling writes for close at 1733399869384Obtaining lock to block concurrent updates at 1733399869384Preparing flush snapshotting stores in d5e0da544abea90f443151cdd8418bf7 at 1733399869384Finished memstore snapshotting TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733399869384Flushing stores of TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7. at 1733399869385 (+1 ms)Flushing d5e0da544abea90f443151cdd8418bf7/A: creating writer at 1733399869385Flushing d5e0da544abea90f443151cdd8418bf7/A: appending metadata at 1733399869388 (+3 ms)Flushing d5e0da544abea90f443151cdd8418bf7/A: closing flushed file at 1733399869388Flushing d5e0da544abea90f443151cdd8418bf7/B: creating writer at 1733399869796 (+408 ms)Flushing d5e0da544abea90f443151cdd8418bf7/B: appending metadata at 1733399869800 (+4 ms)Flushing d5e0da544abea90f443151cdd8418bf7/B: closing flushed file at 1733399869800Flushing d5e0da544abea90f443151cdd8418bf7/C: creating writer at 1733399870210 (+410 ms)Flushing d5e0da544abea90f443151cdd8418bf7/C: appending metadata at 1733399870213 (+3 ms)Flushing d5e0da544abea90f443151cdd8418bf7/C: closing flushed file at 1733399870213Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e31dd79: reopening flushed file at 1733399870621 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37f90c7a: reopening flushed file at 1733399870626 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@537a5981: reopening flushed file at 1733399870631 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d5e0da544abea90f443151cdd8418bf7 in 1252ms, sequenceid=417, compaction requested=true at 1733399870636 (+5 ms)Writing region close event to WAL at 1733399870757 (+121 ms)Running coprocessor post-close hooks at 1733399870760 (+3 ms)Closed at 1733399870760 2024-12-05T11:57:50,762 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:50,763 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=d5e0da544abea90f443151cdd8418bf7, regionState=CLOSED 2024-12-05T11:57:50,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:57:50,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-05T11:57:50,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure d5e0da544abea90f443151cdd8418bf7, server=80666a11a09f,40121,1733399769572 in 1.5480 sec 2024-12-05T11:57:50,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-05T11:57:50,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5e0da544abea90f443151cdd8418bf7, UNASSIGN in 1.5540 sec 2024-12-05T11:57:50,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-05T11:57:50,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5580 sec 2024-12-05T11:57:50,788 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399870788"}]},"ts":"1733399870788"} 2024-12-05T11:57:50,789 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T11:57:50,789 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T11:57:50,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5690 sec 2024-12-05T11:57:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T11:57:51,368 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:51,368 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T11:57:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:51,370 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:51,371 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-05T11:57:51,373 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:51,376 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/recovered.edits] 2024-12-05T11:57:51,379 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/38d8644a22bc4e90b6bc118c0a8cd034 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/38d8644a22bc4e90b6bc118c0a8cd034 2024-12-05T11:57:51,380 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/77d89a6b049c4777b7ba2a66c72230ec to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/77d89a6b049c4777b7ba2a66c72230ec 2024-12-05T11:57:51,381 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/bdf8f82e62de43c08a98812649256973 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/A/bdf8f82e62de43c08a98812649256973 2024-12-05T11:57:51,384 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/2fe36fd4494f40c4a294e254928d8e5c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/2fe36fd4494f40c4a294e254928d8e5c 2024-12-05T11:57:51,385 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/4349bcbbe9994b83bb0f4e6298a6c6da to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/4349bcbbe9994b83bb0f4e6298a6c6da 2024-12-05T11:57:51,386 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/69aef949db9646ce82f2c780761fefda to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/B/69aef949db9646ce82f2c780761fefda 2024-12-05T11:57:51,388 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/9cea0bb3c9044fbe9cd43daa935746f4 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/9cea0bb3c9044fbe9cd43daa935746f4 2024-12-05T11:57:51,389 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/e66203f356584affa1cb437fd6752e06 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/e66203f356584affa1cb437fd6752e06 2024-12-05T11:57:51,390 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/f581201e37a440dba2bd391cb4e74122 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/C/f581201e37a440dba2bd391cb4e74122 2024-12-05T11:57:51,394 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/recovered.edits/420.seqid to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7/recovered.edits/420.seqid 2024-12-05T11:57:51,394 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/d5e0da544abea90f443151cdd8418bf7 2024-12-05T11:57:51,394 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T11:57:51,397 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:51,401 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T11:57:51,404 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T11:57:51,406 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:51,406 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T11:57:51,406 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733399871406"}]},"ts":"9223372036854775807"} 2024-12-05T11:57:51,408 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T11:57:51,409 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d5e0da544abea90f443151cdd8418bf7, NAME => 'TestAcidGuarantees,,1733399836433.d5e0da544abea90f443151cdd8418bf7.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T11:57:51,409 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T11:57:51,409 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733399871409"}]},"ts":"9223372036854775807"} 2024-12-05T11:57:51,411 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T11:57:51,412 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:51,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-12-05T11:57:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-05T11:57:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T11:57:51,480 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:51,488 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=237 (was 238), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=276 (was 272) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7750 (was 6702) - AvailableMemoryMB LEAK? - 2024-12-05T11:57:51,498 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=276, ProcessCount=11, AvailableMemoryMB=7749 2024-12-05T11:57:51,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:57:51,499 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:57:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:51,501 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T11:57:51,501 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:51,501 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-12-05T11:57:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T11:57:51,502 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T11:57:51,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742142_1318 (size=959) 2024-12-05T11:57:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T11:57:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T11:57:51,912 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c 2024-12-05T11:57:51,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742143_1319 (size=53) 2024-12-05T11:57:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T11:57:52,317 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:57:52,318 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6fb7227437f8ecf2f0f7089b8972c742, disabling compactions & flushes 2024-12-05T11:57:52,318 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:52,318 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:52,318 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. after waiting 0 ms 2024-12-05T11:57:52,318 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:52,318 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:52,318 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6fb7227437f8ecf2f0f7089b8972c742: Waiting for close lock at 1733399872318Disabling compacts and flushes for region at 1733399872318Disabling writes for close at 1733399872318Writing region close event to WAL at 1733399872318Closed at 1733399872318 2024-12-05T11:57:52,319 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T11:57:52,319 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733399872319"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733399872319"}]},"ts":"1733399872319"} 2024-12-05T11:57:52,320 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T11:57:52,321 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T11:57:52,321 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399872321"}]},"ts":"1733399872321"} 2024-12-05T11:57:52,323 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T11:57:52,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, ASSIGN}] 2024-12-05T11:57:52,324 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, ASSIGN 2024-12-05T11:57:52,325 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, ASSIGN; state=OFFLINE, location=80666a11a09f,40121,1733399769572; forceNewPlan=false, retain=false 2024-12-05T11:57:52,476 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=6fb7227437f8ecf2f0f7089b8972c742, regionState=OPENING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:57:52,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, ASSIGN because future has completed 2024-12-05T11:57:52,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:57:52,633 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:52,633 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:57:52,634 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,634 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:57:52,634 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,634 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,635 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,636 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fb7227437f8ecf2f0f7089b8972c742 columnFamilyName A 2024-12-05T11:57:52,636 DEBUG [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:52,637 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] regionserver.HStore(327): Store=6fb7227437f8ecf2f0f7089b8972c742/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:57:52,637 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T11:57:52,638 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fb7227437f8ecf2f0f7089b8972c742 columnFamilyName B 2024-12-05T11:57:52,638 DEBUG [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:52,638 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] regionserver.HStore(327): Store=6fb7227437f8ecf2f0f7089b8972c742/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:57:52,638 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,639 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fb7227437f8ecf2f0f7089b8972c742 columnFamilyName C 2024-12-05T11:57:52,639 DEBUG [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:52,640 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] regionserver.HStore(327): Store=6fb7227437f8ecf2f0f7089b8972c742/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:57:52,640 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,641 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,641 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,642 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,642 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,642 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:57:52,643 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,645 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:57:52,645 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 6fb7227437f8ecf2f0f7089b8972c742; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73474906, jitterRate=0.09486141800880432}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:57:52,645 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:52,646 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 6fb7227437f8ecf2f0f7089b8972c742: Running coprocessor pre-open hook at 1733399872634Writing region info on filesystem at 1733399872634Initializing all the Stores at 1733399872635 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399872635Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399872635Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399872635Cleaning up temporary data from old regions at 1733399872642 (+7 ms)Running coprocessor post-open hooks at 1733399872645 (+3 ms)Region opened successfully at 1733399872646 (+1 ms) 2024-12-05T11:57:52,646 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., pid=89, masterSystemTime=1733399872630 2024-12-05T11:57:52,648 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:52,648 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:52,649 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=6fb7227437f8ecf2f0f7089b8972c742, regionState=OPEN, openSeqNum=2, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:57:52,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:57:52,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-05T11:57:52,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 in 173 msec 2024-12-05T11:57:52,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-05T11:57:52,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, ASSIGN in 330 msec 2024-12-05T11:57:52,654 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T11:57:52,655 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399872654"}]},"ts":"1733399872654"} 2024-12-05T11:57:52,656 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T11:57:52,656 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T11:57:52,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1570 sec 2024-12-05T11:57:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T11:57:53,649 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:53,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:53,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:53,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:53,651 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7994aa7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:53,653 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@518814e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:53,655 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:53,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:53,658 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:57:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 80666a11a09f,33741,1733399768894 2024-12-05T11:57:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d997a09 2024-12-05T11:57:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:57:53,660 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:57:53,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:57:53,662 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:57:53,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T11:57:53,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742144_1320 (size=995) 2024-12-05T11:57:54,075 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-05T11:57:54,075 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-05T11:57:54,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:57:54,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, REOPEN/MOVE}] 2024-12-05T11:57:54,080 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, REOPEN/MOVE 2024-12-05T11:57:54,081 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=6fb7227437f8ecf2f0f7089b8972c742, regionState=CLOSING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, REOPEN/MOVE because future has completed 2024-12-05T11:57:54,082 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:57:54,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:57:54,233 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,234 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:57:54,234 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 6fb7227437f8ecf2f0f7089b8972c742, disabling compactions & flushes 2024-12-05T11:57:54,234 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:54,234 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:54,234 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. after waiting 0 ms 2024-12-05T11:57:54,234 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:54,237 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T11:57:54,238 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:54,238 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 6fb7227437f8ecf2f0f7089b8972c742: Waiting for close lock at 1733399874234Running coprocessor pre-close hooks at 1733399874234Disabling compacts and flushes for region at 1733399874234Disabling writes for close at 1733399874234Writing region close event to WAL at 1733399874234Running coprocessor post-close hooks at 1733399874238 (+4 ms)Closed at 1733399874238 2024-12-05T11:57:54,238 WARN [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 6fb7227437f8ecf2f0f7089b8972c742 to self. 2024-12-05T11:57:54,240 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,241 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=6fb7227437f8ecf2f0f7089b8972c742, regionState=CLOSED 2024-12-05T11:57:54,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:57:54,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-05T11:57:54,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 in 161 msec 2024-12-05T11:57:54,246 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, REOPEN/MOVE; state=CLOSED, location=80666a11a09f,40121,1733399769572; forceNewPlan=false, retain=true 2024-12-05T11:57:54,396 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=6fb7227437f8ecf2f0f7089b8972c742, regionState=OPENING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, REOPEN/MOVE because future has completed 2024-12-05T11:57:54,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:57:54,553 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:54,554 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:57:54,554 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,554 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:57:54,554 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,554 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,556 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,557 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fb7227437f8ecf2f0f7089b8972c742 columnFamilyName A 2024-12-05T11:57:54,559 DEBUG [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:54,560 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] regionserver.HStore(327): Store=6fb7227437f8ecf2f0f7089b8972c742/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:57:54,561 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,562 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fb7227437f8ecf2f0f7089b8972c742 columnFamilyName B 2024-12-05T11:57:54,562 DEBUG [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:54,563 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] regionserver.HStore(327): Store=6fb7227437f8ecf2f0f7089b8972c742/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:57:54,563 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,564 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fb7227437f8ecf2f0f7089b8972c742 columnFamilyName C 2024-12-05T11:57:54,564 DEBUG [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:54,564 INFO [StoreOpener-6fb7227437f8ecf2f0f7089b8972c742-1 {}] regionserver.HStore(327): Store=6fb7227437f8ecf2f0f7089b8972c742/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:57:54,564 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,565 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,566 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,566 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,566 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,567 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:57:54,568 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,568 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 6fb7227437f8ecf2f0f7089b8972c742; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62893926, jitterRate=-0.06280747056007385}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:57:54,568 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,569 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 6fb7227437f8ecf2f0f7089b8972c742: Running coprocessor pre-open hook at 1733399874555Writing region info on filesystem at 1733399874555Initializing all the Stores at 1733399874556 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399874556Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399874556Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399874556Cleaning up temporary data from old regions at 1733399874566 (+10 ms)Running coprocessor post-open hooks at 1733399874568 (+2 ms)Region opened successfully at 1733399874569 (+1 ms) 2024-12-05T11:57:54,569 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., pid=94, masterSystemTime=1733399874549 2024-12-05T11:57:54,571 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:54,571 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:54,572 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=6fb7227437f8ecf2f0f7089b8972c742, regionState=OPEN, openSeqNum=5, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:57:54,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-12-05T11:57:54,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 in 175 msec 2024-12-05T11:57:54,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-05T11:57:54,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, REOPEN/MOVE in 495 msec 2024-12-05T11:57:54,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-05T11:57:54,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-12-05T11:57:54,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 917 msec 2024-12-05T11:57:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-05T11:57:54,581 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:54,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:54,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:54,583 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3335bcf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:54,584 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37744eab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,585 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:54,585 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:54,586 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73bf6e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:54,587 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f1e204a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:54,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:54,589 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c53a6bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:54,591 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@156a9a05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:54,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:54,594 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc378be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:54,595 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6833a7c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:54,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:54,598 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f21023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:54,600 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65fdc587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:54,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:54,602 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ce1914, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:54,604 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7849e944, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,607 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:54,607 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:54,608 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37c69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:54,609 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cfd24e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,610 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:54,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:54,611 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:54,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:54,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:54,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a72510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:54,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:54,612 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:54,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42141a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,614 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:54,614 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:54,614 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6047d1f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:54,616 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79c69e0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:57:54,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:57:54,618 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:57:54,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:57:54,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:57:54,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2712fa07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:57:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:57:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:57:54,620 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:57:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4d0f0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:57:54,621 DEBUG [Thread-1489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:54,621 DEBUG [Thread-1487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:54,621 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:54,621 DEBUG [Thread-1474 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:54,621 DEBUG [Thread-1483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:54,622 DEBUG [Thread-1481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:54,622 DEBUG [Thread-1483 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:54,622 DEBUG [Thread-1483 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:54,623 DEBUG [Thread-1481 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-05T11:57:54,623 DEBUG [Thread-1481 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:54,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:54,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:54,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:54,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:54,624 DEBUG [Thread-1472 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:57:54,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:54,624 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:54,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:54,624 DEBUG [Thread-1470 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:54,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:57:54,626 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:54,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:54,627 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:57:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:57:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:57:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:57:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:57:54,632 DEBUG [Thread-1476 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:54,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:57:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:54,635 DEBUG [Thread-1478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:54,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:54,635 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:57:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:57:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:57:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:57:54,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:54,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:57:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:57:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:57:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:54,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T11:57:54,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a9e893c1bba84fb2ac319a591517ca83_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399874671/Put/seqid=0 2024-12-05T11:57:54,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399934696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:54,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399934703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:54,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399934704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399934705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399934711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:54,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742145_1321 (size=14594) 2024-12-05T11:57:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:57:54,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:57:54,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:54,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:57:54,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:54,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:54,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:54,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399934808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:54,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399934818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399934818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:54,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399934818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:54,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399934818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:54,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:57:54,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:54,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:57:54,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:54,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:54,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:57:55,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399935018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399935028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399935028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399935028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399935028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:57:55,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:55,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:57:55,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:55,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:55,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:55,116 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:55,120 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a9e893c1bba84fb2ac319a591517ca83_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a9e893c1bba84fb2ac319a591517ca83_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:55,121 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/9f202461f653437e9000dd71bd8a90fb, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:57:55,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/9f202461f653437e9000dd71bd8a90fb is 175, key is test_row_0/A:col10/1733399874671/Put/seqid=0 2024-12-05T11:57:55,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742146_1322 (size=39549) 2024-12-05T11:57:55,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/9f202461f653437e9000dd71bd8a90fb 2024-12-05T11:57:55,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/c7b298b322b4459a9ce5f954aeb105c7 is 50, key is test_row_0/B:col10/1733399874671/Put/seqid=0 2024-12-05T11:57:55,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742147_1323 (size=12001) 2024-12-05T11:57:55,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/c7b298b322b4459a9ce5f954aeb105c7 2024-12-05T11:57:55,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/73210547eff14fffbc0542ecba290257 is 50, key is test_row_0/C:col10/1733399874671/Put/seqid=0 2024-12-05T11:57:55,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742148_1324 (size=12001) 2024-12-05T11:57:55,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:57:55,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:55,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:57:55,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:55,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:55,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:57:55,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399935328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399935337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399935337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399935337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399935338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:57:55,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:55,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:57:55,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:55,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:55,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:55,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:57:55,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:55,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:57:55,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:55,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:55,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:57:55,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/73210547eff14fffbc0542ecba290257 2024-12-05T11:57:55,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/9f202461f653437e9000dd71bd8a90fb as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9f202461f653437e9000dd71bd8a90fb 2024-12-05T11:57:55,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9f202461f653437e9000dd71bd8a90fb, entries=200, sequenceid=17, filesize=38.6 K 2024-12-05T11:57:55,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/c7b298b322b4459a9ce5f954aeb105c7 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/c7b298b322b4459a9ce5f954aeb105c7 2024-12-05T11:57:55,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/c7b298b322b4459a9ce5f954aeb105c7, entries=150, sequenceid=17, filesize=11.7 K 2024-12-05T11:57:55,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/73210547eff14fffbc0542ecba290257 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/73210547eff14fffbc0542ecba290257 2024-12-05T11:57:55,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/73210547eff14fffbc0542ecba290257, entries=150, sequenceid=17, filesize=11.7 K 2024-12-05T11:57:55,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6fb7227437f8ecf2f0f7089b8972c742 in 951ms, sequenceid=17, compaction requested=false 2024-12-05T11:57:55,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:57:55,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:57:55,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:55,696 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T11:57:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120570f6a162f56145b4a80b85be4b281d65_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399874703/Put/seqid=0 2024-12-05T11:57:55,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742149_1325 (size=12154) 2024-12-05T11:57:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:55,709 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120570f6a162f56145b4a80b85be4b281d65_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120570f6a162f56145b4a80b85be4b281d65_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/7fd75e0d771440a88d1a9666340a239c, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:57:55,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/7fd75e0d771440a88d1a9666340a239c is 175, key is test_row_0/A:col10/1733399874703/Put/seqid=0 2024-12-05T11:57:55,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742150_1326 (size=30955) 2024-12-05T11:57:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:57:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:55,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:57:55,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399935852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399935853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399935854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399935854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399935859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399935968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399935968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399935968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399935968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399935968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:56,116 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/7fd75e0d771440a88d1a9666340a239c 2024-12-05T11:57:56,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/5029606044184201ae563322ecdbd5fa is 50, key is test_row_0/B:col10/1733399874703/Put/seqid=0 2024-12-05T11:57:56,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742151_1327 (size=12001) 2024-12-05T11:57:56,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399936179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399936179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399936179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399936179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:56,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399936179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:56,288 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:57:56,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399936488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:56,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399936488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:56,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399936488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:56,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399936488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:56,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399936498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:56,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:56,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:56,534 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/5029606044184201ae563322ecdbd5fa 2024-12-05T11:57:56,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/0bfa9189ed1846afb5c33abcd40c17de is 50, key is test_row_0/C:col10/1733399874703/Put/seqid=0 2024-12-05T11:57:56,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742152_1328 (size=12001) 2024-12-05T11:57:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:57:56,951 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/0bfa9189ed1846afb5c33abcd40c17de 2024-12-05T11:57:56,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/7fd75e0d771440a88d1a9666340a239c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/7fd75e0d771440a88d1a9666340a239c 2024-12-05T11:57:56,958 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/7fd75e0d771440a88d1a9666340a239c, entries=150, sequenceid=40, filesize=30.2 K 2024-12-05T11:57:56,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/5029606044184201ae563322ecdbd5fa as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/5029606044184201ae563322ecdbd5fa 2024-12-05T11:57:56,962 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/5029606044184201ae563322ecdbd5fa, entries=150, sequenceid=40, filesize=11.7 K 2024-12-05T11:57:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/0bfa9189ed1846afb5c33abcd40c17de as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/0bfa9189ed1846afb5c33abcd40c17de 2024-12-05T11:57:56,966 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/0bfa9189ed1846afb5c33abcd40c17de, entries=150, sequenceid=40, filesize=11.7 K 2024-12-05T11:57:56,967 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 6fb7227437f8ecf2f0f7089b8972c742 in 1271ms, sequenceid=40, compaction requested=false 2024-12-05T11:57:56,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:57:56,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:56,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-05T11:57:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-05T11:57:56,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-05T11:57:56,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3430 sec 2024-12-05T11:57:56,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.3510 sec 2024-12-05T11:57:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:57,000 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:57,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T11:57:57,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057974883a774e4d448648c14d458309e9_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399875853/Put/seqid=0 2024-12-05T11:57:57,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742153_1329 (size=14594) 2024-12-05T11:57:57,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399937047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399937053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399937057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399937059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399937060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399937168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399937168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399937178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399937178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399937178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399937378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399937378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399937387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399937387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399937387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,427 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:57,430 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057974883a774e4d448648c14d458309e9_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057974883a774e4d448648c14d458309e9_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:57,430 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/2358756ad04a48938e43a07184f45419, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:57:57,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/2358756ad04a48938e43a07184f45419 is 175, key is test_row_0/A:col10/1733399875853/Put/seqid=0 2024-12-05T11:57:57,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742154_1330 (size=39549) 2024-12-05T11:57:57,435 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/2358756ad04a48938e43a07184f45419 2024-12-05T11:57:57,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/d6bd84fbf16b4e978f122578b1760d13 is 50, key is test_row_0/B:col10/1733399875853/Put/seqid=0 2024-12-05T11:57:57,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742155_1331 (size=12001) 2024-12-05T11:57:57,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399937688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399937688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399937698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399937708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399937708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:57,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/d6bd84fbf16b4e978f122578b1760d13 2024-12-05T11:57:57,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/1fcd2dcd451141808119d77d9730a83a is 50, key is test_row_0/C:col10/1733399875853/Put/seqid=0 2024-12-05T11:57:57,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742156_1332 (size=12001) 2024-12-05T11:57:58,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:58,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399938198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:58,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:58,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399938198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:58,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:58,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399938208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:58,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:58,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399938218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399938218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:58,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/1fcd2dcd451141808119d77d9730a83a 2024-12-05T11:57:58,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/2358756ad04a48938e43a07184f45419 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/2358756ad04a48938e43a07184f45419 2024-12-05T11:57:58,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/2358756ad04a48938e43a07184f45419, entries=200, sequenceid=54, filesize=38.6 K 2024-12-05T11:57:58,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/d6bd84fbf16b4e978f122578b1760d13 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d6bd84fbf16b4e978f122578b1760d13 2024-12-05T11:57:58,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d6bd84fbf16b4e978f122578b1760d13, entries=150, sequenceid=54, filesize=11.7 K 2024-12-05T11:57:58,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/1fcd2dcd451141808119d77d9730a83a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/1fcd2dcd451141808119d77d9730a83a 2024-12-05T11:57:58,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/1fcd2dcd451141808119d77d9730a83a, entries=150, sequenceid=54, filesize=11.7 K 2024-12-05T11:57:58,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 6fb7227437f8ecf2f0f7089b8972c742 in 1276ms, sequenceid=54, compaction requested=true 2024-12-05T11:57:58,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:57:58,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:57:58,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:58,277 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:58,277 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:58,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:57:58,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:58,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:57:58,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:58,279 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:58,279 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:58,279 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/A is initiating minor compaction (all files) 2024-12-05T11:57:58,279 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/B is initiating minor compaction (all files) 2024-12-05T11:57:58,279 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/A in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:58,279 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/B in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:58,280 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/c7b298b322b4459a9ce5f954aeb105c7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/5029606044184201ae563322ecdbd5fa, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d6bd84fbf16b4e978f122578b1760d13] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=35.2 K 2024-12-05T11:57:58,280 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9f202461f653437e9000dd71bd8a90fb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/7fd75e0d771440a88d1a9666340a239c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/2358756ad04a48938e43a07184f45419] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=107.5 K 2024-12-05T11:57:58,280 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:58,280 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9f202461f653437e9000dd71bd8a90fb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/7fd75e0d771440a88d1a9666340a239c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/2358756ad04a48938e43a07184f45419] 2024-12-05T11:57:58,280 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c7b298b322b4459a9ce5f954aeb105c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733399874634 2024-12-05T11:57:58,280 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f202461f653437e9000dd71bd8a90fb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733399874634 2024-12-05T11:57:58,280 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5029606044184201ae563322ecdbd5fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733399874702 2024-12-05T11:57:58,280 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fd75e0d771440a88d1a9666340a239c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733399874702 2024-12-05T11:57:58,280 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d6bd84fbf16b4e978f122578b1760d13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733399875853 2024-12-05T11:57:58,281 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2358756ad04a48938e43a07184f45419, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733399875852 2024-12-05T11:57:58,296 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:58,297 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/0da966a1fe9c4a1baa81443100cf2920 is 50, key is test_row_0/B:col10/1733399875853/Put/seqid=0 2024-12-05T11:57:58,298 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:57:58,319 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205b4d3407ac35e4e609356c89f174347c5_6fb7227437f8ecf2f0f7089b8972c742 store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:57:58,321 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205b4d3407ac35e4e609356c89f174347c5_6fb7227437f8ecf2f0f7089b8972c742, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:57:58,321 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b4d3407ac35e4e609356c89f174347c5_6fb7227437f8ecf2f0f7089b8972c742 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:57:58,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742157_1333 (size=12104) 2024-12-05T11:57:58,346 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/0da966a1fe9c4a1baa81443100cf2920 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/0da966a1fe9c4a1baa81443100cf2920 2024-12-05T11:57:58,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742158_1334 (size=4469) 2024-12-05T11:57:58,359 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/B of 6fb7227437f8ecf2f0f7089b8972c742 into 0da966a1fe9c4a1baa81443100cf2920(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:58,359 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:57:58,359 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/B, priority=13, startTime=1733399878277; duration=0sec 2024-12-05T11:57:58,360 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:57:58,360 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:B 2024-12-05T11:57:58,360 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:57:58,363 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:57:58,363 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/C is initiating minor compaction (all files) 2024-12-05T11:57:58,363 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/C in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:58,363 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/73210547eff14fffbc0542ecba290257, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/0bfa9189ed1846afb5c33abcd40c17de, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/1fcd2dcd451141808119d77d9730a83a] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=35.2 K 2024-12-05T11:57:58,363 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 73210547eff14fffbc0542ecba290257, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733399874634 2024-12-05T11:57:58,364 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bfa9189ed1846afb5c33abcd40c17de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733399874702 2024-12-05T11:57:58,364 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fcd2dcd451141808119d77d9730a83a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733399875853 2024-12-05T11:57:58,374 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:58,374 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/a38b14b75c844ece8342f18381f20f12 is 50, key is test_row_0/C:col10/1733399875853/Put/seqid=0 2024-12-05T11:57:58,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742159_1335 (size=12104) 2024-12-05T11:57:58,760 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#A#compaction#286 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:57:58,760 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/960b053d219444c8a579b6c8fccd1ce6 is 175, key is test_row_0/A:col10/1733399875853/Put/seqid=0 2024-12-05T11:57:58,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742160_1336 (size=31058) 2024-12-05T11:57:58,785 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/a38b14b75c844ece8342f18381f20f12 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/a38b14b75c844ece8342f18381f20f12 2024-12-05T11:57:58,790 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/C of 6fb7227437f8ecf2f0f7089b8972c742 into a38b14b75c844ece8342f18381f20f12(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:58,790 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:57:58,790 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/C, priority=13, startTime=1733399878277; duration=0sec 2024-12-05T11:57:58,790 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:58,791 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:C 2024-12-05T11:57:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:57:58,798 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:58,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-05T11:57:58,800 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T11:57:58,801 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:58,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T11:57:58,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T11:57:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:58,953 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T11:57:58,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ce38e928e0694bbf8e6c096f342a0738_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399877055/Put/seqid=0 2024-12-05T11:57:58,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742161_1337 (size=12154) 2024-12-05T11:57:58,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:58,965 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ce38e928e0694bbf8e6c096f342a0738_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ce38e928e0694bbf8e6c096f342a0738_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:58,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/57c3c867711f4669a74c96bd06ff3ccb, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:57:58,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/57c3c867711f4669a74c96bd06ff3ccb is 175, key is test_row_0/A:col10/1733399877055/Put/seqid=0 2024-12-05T11:57:58,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742162_1338 (size=30955) 2024-12-05T11:57:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T11:57:59,175 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/960b053d219444c8a579b6c8fccd1ce6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/960b053d219444c8a579b6c8fccd1ce6 2024-12-05T11:57:59,184 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/A of 6fb7227437f8ecf2f0f7089b8972c742 into 960b053d219444c8a579b6c8fccd1ce6(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:57:59,184 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:57:59,185 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/A, priority=13, startTime=1733399878277; duration=0sec 2024-12-05T11:57:59,185 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:57:59,185 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:A 2024-12-05T11:57:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:59,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:57:59,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399939238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399939239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,247 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in 6fb7227437f8ecf2f0f7089b8972c742 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,248 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in 6fb7227437f8ecf2f0f7089b8972c742 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:57:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399939246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399939247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399939249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,370 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/57c3c867711f4669a74c96bd06ff3ccb 2024-12-05T11:57:59,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399939368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399939368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399939368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/e85095716a1a49aeb1777eafee423ab2 is 50, key is test_row_0/B:col10/1733399877055/Put/seqid=0 2024-12-05T11:57:59,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742163_1339 (size=12001) 2024-12-05T11:57:59,386 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/e85095716a1a49aeb1777eafee423ab2 2024-12-05T11:57:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/b37dfc4a5202465d8f25ee5d6e3670ae is 50, key is test_row_0/C:col10/1733399877055/Put/seqid=0 2024-12-05T11:57:59,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742164_1340 (size=12001) 2024-12-05T11:57:59,404 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/b37dfc4a5202465d8f25ee5d6e3670ae 2024-12-05T11:57:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/57c3c867711f4669a74c96bd06ff3ccb as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/57c3c867711f4669a74c96bd06ff3ccb 2024-12-05T11:57:59,417 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/57c3c867711f4669a74c96bd06ff3ccb, entries=150, sequenceid=79, filesize=30.2 K 2024-12-05T11:57:59,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/e85095716a1a49aeb1777eafee423ab2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/e85095716a1a49aeb1777eafee423ab2 2024-12-05T11:57:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,421 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/e85095716a1a49aeb1777eafee423ab2, entries=150, sequenceid=79, filesize=11.7 K 2024-12-05T11:57:59,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/b37dfc4a5202465d8f25ee5d6e3670ae as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/b37dfc4a5202465d8f25ee5d6e3670ae 2024-12-05T11:57:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,425 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/b37dfc4a5202465d8f25ee5d6e3670ae, entries=150, sequenceid=79, filesize=11.7 K 2024-12-05T11:57:59,426 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 6fb7227437f8ecf2f0f7089b8972c742 in 473ms, sequenceid=79, compaction requested=false 2024-12-05T11:57:59,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:57:59,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:57:59,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-05T11:57:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-05T11:57:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T11:57:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-05T11:57:59,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 626 msec 2024-12-05T11:57:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 632 msec 2024-12-05T11:57:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:57:59,592 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:57:59,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-05T11:57:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120503717168a55c498c953f561a3e406160_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399879244/Put/seqid=0 2024-12-05T11:57:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742165_1341 (size=12154) 2024-12-05T11:57:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:57:59,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399939662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399939662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399939671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399939778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399939778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399939788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T11:57:59,938 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:57:59,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:57:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-05T11:57:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:57:59,940 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:57:59,940 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:57:59,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:57:59,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399939988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:57:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399939988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:57:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:57:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:57:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:00,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399939998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:00,037 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:00,040 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120503717168a55c498c953f561a3e406160_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120503717168a55c498c953f561a3e406160_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:00,040 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/82138d561955465d962e5e8f0b4ac797, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:00,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/82138d561955465d962e5e8f0b4ac797 is 175, key is test_row_0/A:col10/1733399879244/Put/seqid=0 2024-12-05T11:58:00,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742166_1342 (size=30951) 2024-12-05T11:58:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:58:00,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:58:00,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:00,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:00,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:00,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:58:00,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:00,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:00,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:00,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:58:00,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399940297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:00,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399940298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:00,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399940308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:00,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:58:00,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:00,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:00,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:00,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,445 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/82138d561955465d962e5e8f0b4ac797 2024-12-05T11:58:00,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/6d8d723db96d466698e16d210dd3ad2c is 50, key is test_row_0/B:col10/1733399879244/Put/seqid=0 2024-12-05T11:58:00,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742167_1343 (size=9657) 2024-12-05T11:58:00,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:58:00,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:00,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:00,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:00,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:58:00,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:58:00,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:00,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:00,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:00,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399940808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:00,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399940808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:00,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399940818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:00,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/6d8d723db96d466698e16d210dd3ad2c 2024-12-05T11:58:00,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:58:00,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:00,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:00,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:00,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:00,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/bfe131bbfc2e4d6db452138217f32ef0 is 50, key is test_row_0/C:col10/1733399879244/Put/seqid=0 2024-12-05T11:58:00,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742168_1344 (size=9657) 2024-12-05T11:58:01,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:58:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:01,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:58:01,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:58:01,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:01,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:01,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:01,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:01,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:01,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399941257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:01,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399941257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:01,265 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in 6fb7227437f8ecf2f0f7089b8972c742 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:01,265 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in 6fb7227437f8ecf2f0f7089b8972c742 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:01,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/bfe131bbfc2e4d6db452138217f32ef0 2024-12-05T11:58:01,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/82138d561955465d962e5e8f0b4ac797 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/82138d561955465d962e5e8f0b4ac797 2024-12-05T11:58:01,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/82138d561955465d962e5e8f0b4ac797, entries=150, sequenceid=95, filesize=30.2 K 2024-12-05T11:58:01,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/6d8d723db96d466698e16d210dd3ad2c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6d8d723db96d466698e16d210dd3ad2c 2024-12-05T11:58:01,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6d8d723db96d466698e16d210dd3ad2c, entries=100, sequenceid=95, filesize=9.4 K 2024-12-05T11:58:01,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/bfe131bbfc2e4d6db452138217f32ef0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/bfe131bbfc2e4d6db452138217f32ef0 2024-12-05T11:58:01,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/bfe131bbfc2e4d6db452138217f32ef0, entries=100, sequenceid=95, filesize=9.4 K 2024-12-05T11:58:01,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6fb7227437f8ecf2f0f7089b8972c742 in 1708ms, sequenceid=95, compaction requested=true 2024-12-05T11:58:01,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:01,301 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:01,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:01,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:01,301 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:01,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:01,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:01,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:01,302 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:01,302 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/B is initiating minor compaction (all files) 2024-12-05T11:58:01,302 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/B in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:01,302 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/0da966a1fe9c4a1baa81443100cf2920, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/e85095716a1a49aeb1777eafee423ab2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6d8d723db96d466698e16d210dd3ad2c] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=33.0 K 2024-12-05T11:58:01,302 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:01,302 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/A is initiating minor compaction (all files) 2024-12-05T11:58:01,302 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/A in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:01,302 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/960b053d219444c8a579b6c8fccd1ce6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/57c3c867711f4669a74c96bd06ff3ccb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/82138d561955465d962e5e8f0b4ac797] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=90.8 K 2024-12-05T11:58:01,302 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:01,302 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/960b053d219444c8a579b6c8fccd1ce6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/57c3c867711f4669a74c96bd06ff3ccb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/82138d561955465d962e5e8f0b4ac797] 2024-12-05T11:58:01,303 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0da966a1fe9c4a1baa81443100cf2920, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733399875853 2024-12-05T11:58:01,303 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 960b053d219444c8a579b6c8fccd1ce6, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733399875853 2024-12-05T11:58:01,303 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting e85095716a1a49aeb1777eafee423ab2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733399877055 2024-12-05T11:58:01,303 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57c3c867711f4669a74c96bd06ff3ccb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733399877055 2024-12-05T11:58:01,303 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d8d723db96d466698e16d210dd3ad2c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733399879244 2024-12-05T11:58:01,304 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82138d561955465d962e5e8f0b4ac797, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733399879231 2024-12-05T11:58:01,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:01,316 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:01,317 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/27c26b83649345ac96c5105f574def4b is 50, key is test_row_0/B:col10/1733399879244/Put/seqid=0 2024-12-05T11:58:01,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:58:01,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:01,319 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T11:58:01,327 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:01,339 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412051aef8c3bf5b7463abfda8726229b62b9_6fb7227437f8ecf2f0f7089b8972c742 store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:01,341 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412051aef8c3bf5b7463abfda8726229b62b9_6fb7227437f8ecf2f0f7089b8972c742, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:01,341 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051aef8c3bf5b7463abfda8726229b62b9_6fb7227437f8ecf2f0f7089b8972c742 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:01,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e2f3cee715ce4c4d853dc829960ab91f_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399879669/Put/seqid=0 2024-12-05T11:58:01,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742169_1345 (size=12207) 2024-12-05T11:58:01,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742170_1346 (size=4469) 2024-12-05T11:58:01,373 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/27c26b83649345ac96c5105f574def4b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/27c26b83649345ac96c5105f574def4b 2024-12-05T11:58:01,379 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/B of 6fb7227437f8ecf2f0f7089b8972c742 into 27c26b83649345ac96c5105f574def4b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:01,379 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:01,379 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/B, priority=13, startTime=1733399881301; duration=0sec 2024-12-05T11:58:01,379 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:01,379 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:B 2024-12-05T11:58:01,379 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:01,382 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:01,382 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/C is initiating minor compaction (all files) 2024-12-05T11:58:01,382 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/C in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:01,382 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/a38b14b75c844ece8342f18381f20f12, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/b37dfc4a5202465d8f25ee5d6e3670ae, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/bfe131bbfc2e4d6db452138217f32ef0] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=33.0 K 2024-12-05T11:58:01,383 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting a38b14b75c844ece8342f18381f20f12, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733399875853 2024-12-05T11:58:01,383 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting b37dfc4a5202465d8f25ee5d6e3670ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733399877055 2024-12-05T11:58:01,383 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe131bbfc2e4d6db452138217f32ef0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733399879244 2024-12-05T11:58:01,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742171_1347 (size=12154) 2024-12-05T11:58:01,397 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#C#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:01,398 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/2f09de2f81ee4b868ed4e5df78a3b543 is 50, key is test_row_0/C:col10/1733399879244/Put/seqid=0 2024-12-05T11:58:01,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742172_1348 (size=12207) 2024-12-05T11:58:01,770 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#A#compaction#295 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:01,771 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/ca09db1599124239b45a5855819aeb04 is 175, key is test_row_0/A:col10/1733399879244/Put/seqid=0 2024-12-05T11:58:01,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742173_1349 (size=31268) 2024-12-05T11:58:01,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:01,798 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e2f3cee715ce4c4d853dc829960ab91f_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e2f3cee715ce4c4d853dc829960ab91f_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:01,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/cdec014816df4acdb629fe9683cd6f35, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:01,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/cdec014816df4acdb629fe9683cd6f35 is 175, key is test_row_0/A:col10/1733399879669/Put/seqid=0 2024-12-05T11:58:01,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742174_1350 (size=30955) 2024-12-05T11:58:01,813 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/cdec014816df4acdb629fe9683cd6f35 2024-12-05T11:58:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:01,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/ec53497457f342d08367031d8f40edb1 is 50, key is test_row_0/B:col10/1733399879669/Put/seqid=0 2024-12-05T11:58:01,826 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/2f09de2f81ee4b868ed4e5df78a3b543 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/2f09de2f81ee4b868ed4e5df78a3b543 2024-12-05T11:58:01,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742175_1351 (size=12001) 2024-12-05T11:58:01,833 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/ec53497457f342d08367031d8f40edb1 2024-12-05T11:58:01,835 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/C of 6fb7227437f8ecf2f0f7089b8972c742 into 2f09de2f81ee4b868ed4e5df78a3b543(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:01,835 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:01,835 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/C, priority=13, startTime=1733399881301; duration=0sec 2024-12-05T11:58:01,835 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:01,835 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:C 2024-12-05T11:58:01,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/96d9305763f34392b8b77b510b10228c is 50, key is test_row_0/C:col10/1733399879669/Put/seqid=0 2024-12-05T11:58:01,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742176_1352 (size=12001) 2024-12-05T11:58:01,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399941854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:01,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399941855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:01,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:01,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399941857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:01,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:01,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399941968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:01,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399941968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:01,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399941968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:58:02,178 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/ca09db1599124239b45a5855819aeb04 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/ca09db1599124239b45a5855819aeb04 2024-12-05T11:58:02,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399942178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:02,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399942178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,182 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/A of 6fb7227437f8ecf2f0f7089b8972c742 into ca09db1599124239b45a5855819aeb04(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:02,182 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:02,182 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/A, priority=13, startTime=1733399881300; duration=0sec 2024-12-05T11:58:02,182 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:02,182 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:A 2024-12-05T11:58:02,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399942179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,250 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/96d9305763f34392b8b77b510b10228c 2024-12-05T11:58:02,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/cdec014816df4acdb629fe9683cd6f35 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/cdec014816df4acdb629fe9683cd6f35 2024-12-05T11:58:02,257 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/cdec014816df4acdb629fe9683cd6f35, entries=150, sequenceid=116, filesize=30.2 K 2024-12-05T11:58:02,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/ec53497457f342d08367031d8f40edb1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/ec53497457f342d08367031d8f40edb1 2024-12-05T11:58:02,260 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/ec53497457f342d08367031d8f40edb1, entries=150, sequenceid=116, filesize=11.7 K 2024-12-05T11:58:02,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/96d9305763f34392b8b77b510b10228c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/96d9305763f34392b8b77b510b10228c 2024-12-05T11:58:02,265 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/96d9305763f34392b8b77b510b10228c, entries=150, sequenceid=116, filesize=11.7 K 2024-12-05T11:58:02,266 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6fb7227437f8ecf2f0f7089b8972c742 in 947ms, sequenceid=116, compaction requested=false 2024-12-05T11:58:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-05T11:58:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-05T11:58:02,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-05T11:58:02,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3280 sec 2024-12-05T11:58:02,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.3320 sec 2024-12-05T11:58:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:02,491 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:02,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-05T11:58:02,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057c740b3e2b7e44f9b4ece09a82ccc09e_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399882491/Put/seqid=0 2024-12-05T11:58:02,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742177_1353 (size=14794) 2024-12-05T11:58:02,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399942515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:02,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399942520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:02,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399942520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399942628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399942628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399942628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399942838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:02,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399942838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399942838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:02,908 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:02,910 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057c740b3e2b7e44f9b4ece09a82ccc09e_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057c740b3e2b7e44f9b4ece09a82ccc09e_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:02,911 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/729520e0a9214037b854d3bd9fc5f4b5, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:02,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/729520e0a9214037b854d3bd9fc5f4b5 is 175, key is test_row_0/A:col10/1733399882491/Put/seqid=0 2024-12-05T11:58:02,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742178_1354 (size=39749) 2024-12-05T11:58:03,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399943147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:03,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399943147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:03,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399943148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:03,316 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/729520e0a9214037b854d3bd9fc5f4b5 2024-12-05T11:58:03,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/d14b498445d04bdb854fd29eb6cbd682 is 50, key is test_row_0/B:col10/1733399882491/Put/seqid=0 2024-12-05T11:58:03,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742179_1355 (size=12151) 2024-12-05T11:58:03,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399943657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:03,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399943657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:03,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399943657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:03,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/d14b498445d04bdb854fd29eb6cbd682 2024-12-05T11:58:03,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/13610819ed3f47358120a3ed84eb26b6 is 50, key is test_row_0/C:col10/1733399882491/Put/seqid=0 2024-12-05T11:58:03,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742180_1356 (size=12151) 2024-12-05T11:58:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:58:04,108 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:04,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-05T11:58:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T11:58:04,110 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:04,110 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:04,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:04,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/13610819ed3f47358120a3ed84eb26b6 2024-12-05T11:58:04,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/729520e0a9214037b854d3bd9fc5f4b5 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/729520e0a9214037b854d3bd9fc5f4b5 2024-12-05T11:58:04,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/729520e0a9214037b854d3bd9fc5f4b5, entries=200, sequenceid=137, filesize=38.8 K 2024-12-05T11:58:04,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/d14b498445d04bdb854fd29eb6cbd682 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d14b498445d04bdb854fd29eb6cbd682 2024-12-05T11:58:04,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d14b498445d04bdb854fd29eb6cbd682, entries=150, sequenceid=137, filesize=11.9 K 2024-12-05T11:58:04,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/13610819ed3f47358120a3ed84eb26b6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/13610819ed3f47358120a3ed84eb26b6 2024-12-05T11:58:04,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/13610819ed3f47358120a3ed84eb26b6, entries=150, sequenceid=137, filesize=11.9 K 2024-12-05T11:58:04,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 6fb7227437f8ecf2f0f7089b8972c742 in 1669ms, sequenceid=137, compaction requested=true 2024-12-05T11:58:04,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:04,160 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:04,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:04,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:04,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:04,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:04,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:04,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:04,161 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:04,161 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101972 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:04,162 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/A is initiating minor compaction (all files) 2024-12-05T11:58:04,162 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/A in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:04,162 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/ca09db1599124239b45a5855819aeb04, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/cdec014816df4acdb629fe9683cd6f35, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/729520e0a9214037b854d3bd9fc5f4b5] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=99.6 K 2024-12-05T11:58:04,162 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:04,162 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/ca09db1599124239b45a5855819aeb04, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/cdec014816df4acdb629fe9683cd6f35, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/729520e0a9214037b854d3bd9fc5f4b5] 2024-12-05T11:58:04,162 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:04,162 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/B is initiating minor compaction (all files) 2024-12-05T11:58:04,162 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca09db1599124239b45a5855819aeb04, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733399877058 2024-12-05T11:58:04,162 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/B in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:04,162 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/27c26b83649345ac96c5105f574def4b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/ec53497457f342d08367031d8f40edb1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d14b498445d04bdb854fd29eb6cbd682] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=35.5 K 2024-12-05T11:58:04,162 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdec014816df4acdb629fe9683cd6f35, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733399879640 2024-12-05T11:58:04,163 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 729520e0a9214037b854d3bd9fc5f4b5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733399881855 2024-12-05T11:58:04,163 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 27c26b83649345ac96c5105f574def4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733399877058 2024-12-05T11:58:04,163 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ec53497457f342d08367031d8f40edb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733399879640 2024-12-05T11:58:04,163 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d14b498445d04bdb854fd29eb6cbd682, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733399881855 2024-12-05T11:58:04,170 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:04,175 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#B#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:04,176 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/3727766642f340c0ab928b274a5822b5 is 50, key is test_row_0/B:col10/1733399882491/Put/seqid=0 2024-12-05T11:58:04,176 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412054fb3a19a79cd4e5aa29830f530e7d98a_6fb7227437f8ecf2f0f7089b8972c742 store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:04,177 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412054fb3a19a79cd4e5aa29830f530e7d98a_6fb7227437f8ecf2f0f7089b8972c742, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:04,178 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054fb3a19a79cd4e5aa29830f530e7d98a_6fb7227437f8ecf2f0f7089b8972c742 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:04,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742182_1358 (size=4469) 2024-12-05T11:58:04,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742181_1357 (size=12459) 2024-12-05T11:58:04,197 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/3727766642f340c0ab928b274a5822b5 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/3727766642f340c0ab928b274a5822b5 2024-12-05T11:58:04,204 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/B of 6fb7227437f8ecf2f0f7089b8972c742 into 3727766642f340c0ab928b274a5822b5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:04,204 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:04,204 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/B, priority=13, startTime=1733399884160; duration=0sec 2024-12-05T11:58:04,205 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:04,205 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:B 2024-12-05T11:58:04,205 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:04,209 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:04,209 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/C is initiating minor compaction (all files) 2024-12-05T11:58:04,209 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/C in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:04,209 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/2f09de2f81ee4b868ed4e5df78a3b543, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/96d9305763f34392b8b77b510b10228c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/13610819ed3f47358120a3ed84eb26b6] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=35.5 K 2024-12-05T11:58:04,209 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f09de2f81ee4b868ed4e5df78a3b543, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733399877058 2024-12-05T11:58:04,210 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 96d9305763f34392b8b77b510b10228c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733399879640 2024-12-05T11:58:04,210 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 13610819ed3f47358120a3ed84eb26b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733399881855 2024-12-05T11:58:04,217 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:04,217 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/201f04cf91214e71975e1f127cc36d20 is 50, key is test_row_0/C:col10/1733399882491/Put/seqid=0 2024-12-05T11:58:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T11:58:04,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742183_1359 (size=12459) 2024-12-05T11:58:04,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T11:58:04,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:04,263 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-05T11:58:04,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120541b68d841feb407aa8d3f3e3fbd4d56a_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399882519/Put/seqid=0 2024-12-05T11:58:04,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742184_1360 (size=12304) 2024-12-05T11:58:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T11:58:04,589 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#A#compaction#303 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:04,590 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/106a1c4fc148499b8469a5b8ed819320 is 175, key is test_row_0/A:col10/1733399882491/Put/seqid=0 2024-12-05T11:58:04,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742185_1361 (size=31413) 2024-12-05T11:58:04,605 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/106a1c4fc148499b8469a5b8ed819320 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/106a1c4fc148499b8469a5b8ed819320 2024-12-05T11:58:04,612 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/A of 6fb7227437f8ecf2f0f7089b8972c742 into 106a1c4fc148499b8469a5b8ed819320(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:04,612 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:04,612 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/A, priority=13, startTime=1733399884160; duration=0sec 2024-12-05T11:58:04,613 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:04,613 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:A 2024-12-05T11:58:04,627 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/201f04cf91214e71975e1f127cc36d20 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/201f04cf91214e71975e1f127cc36d20 2024-12-05T11:58:04,632 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/C of 6fb7227437f8ecf2f0f7089b8972c742 into 201f04cf91214e71975e1f127cc36d20(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:04,632 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:04,632 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/C, priority=13, startTime=1733399884161; duration=0sec 2024-12-05T11:58:04,632 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:04,632 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:C 2024-12-05T11:58:04,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:04,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:04,682 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120541b68d841feb407aa8d3f3e3fbd4d56a_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120541b68d841feb407aa8d3f3e3fbd4d56a_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:04,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/f1929850c9ab405bbdbb5c9cb04d3b33, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:04,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/f1929850c9ab405bbdbb5c9cb04d3b33 is 175, key is test_row_0/A:col10/1733399882519/Put/seqid=0 2024-12-05T11:58:04,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742186_1362 (size=31105) 2024-12-05T11:58:04,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399944716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:04,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399944725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:04,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399944726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:04,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T11:58:04,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399944838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:04,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399944839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:04,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399944839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399945048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399945048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399945048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,091 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/f1929850c9ab405bbdbb5c9cb04d3b33 2024-12-05T11:58:05,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/f350b87479cf47a7836a813889486d2b is 50, key is test_row_0/B:col10/1733399882519/Put/seqid=0 2024-12-05T11:58:05,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742187_1363 (size=12151) 2024-12-05T11:58:05,136 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/f350b87479cf47a7836a813889486d2b 2024-12-05T11:58:05,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/eea73ecd03804e62b1aa4636dcad6433 is 50, key is test_row_0/C:col10/1733399882519/Put/seqid=0 2024-12-05T11:58:05,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742188_1364 (size=12151) 2024-12-05T11:58:05,163 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/eea73ecd03804e62b1aa4636dcad6433 2024-12-05T11:58:05,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/f1929850c9ab405bbdbb5c9cb04d3b33 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f1929850c9ab405bbdbb5c9cb04d3b33 2024-12-05T11:58:05,180 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f1929850c9ab405bbdbb5c9cb04d3b33, entries=150, sequenceid=156, filesize=30.4 K 2024-12-05T11:58:05,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/f350b87479cf47a7836a813889486d2b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f350b87479cf47a7836a813889486d2b 2024-12-05T11:58:05,211 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f350b87479cf47a7836a813889486d2b, entries=150, sequenceid=156, filesize=11.9 K 2024-12-05T11:58:05,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/eea73ecd03804e62b1aa4636dcad6433 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/eea73ecd03804e62b1aa4636dcad6433 2024-12-05T11:58:05,231 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/eea73ecd03804e62b1aa4636dcad6433, entries=150, sequenceid=156, filesize=11.9 K 2024-12-05T11:58:05,232 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 6fb7227437f8ecf2f0f7089b8972c742 in 969ms, sequenceid=156, compaction requested=false 2024-12-05T11:58:05,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:05,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:05,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-05T11:58:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-05T11:58:05,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-05T11:58:05,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1240 sec 2024-12-05T11:58:05,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.1320 sec 2024-12-05T11:58:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T11:58:05,249 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:05,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-05T11:58:05,251 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:05,252 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:05,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T11:58:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:05,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-05T11:58:05,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050e5db4aa5ce142fab283ab070babfe31_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399884723/Put/seqid=0 2024-12-05T11:58:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T11:58:05,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742189_1365 (size=14794) 2024-12-05T11:58:05,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399945368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399945367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399945368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399945369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399945379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T11:58:05,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:05,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:05,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:05,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:05,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:05,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399945490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399945490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399945498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T11:58:05,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:05,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:05,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:05,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T11:58:05,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399945698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399945698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T11:58:05,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:05,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:05,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:05,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:05,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:05,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399945717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,766 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:05,771 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050e5db4aa5ce142fab283ab070babfe31_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050e5db4aa5ce142fab283ab070babfe31_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:05,777 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/f8300cc0a5ed4a4f8320816b6658f407, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:05,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/f8300cc0a5ed4a4f8320816b6658f407 is 175, key is test_row_0/A:col10/1733399884723/Put/seqid=0 2024-12-05T11:58:05,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742190_1366 (size=39749) 2024-12-05T11:58:05,811 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/f8300cc0a5ed4a4f8320816b6658f407 2024-12-05T11:58:05,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/0277ca04c61d449085d37dc6d89d2774 is 50, key is test_row_0/B:col10/1733399884723/Put/seqid=0 2024-12-05T11:58:05,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742191_1367 (size=12151) 2024-12-05T11:58:05,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T11:58:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:05,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:05,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T11:58:05,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399945887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399945887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399946008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:06,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399946008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:06,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T11:58:06,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:06,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:06,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:06,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399946028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:06,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:06,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T11:58:06,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:06,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:06,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:06,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/0277ca04c61d449085d37dc6d89d2774 2024-12-05T11:58:06,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/2e42290099d54627ba68c00f0db854f6 is 50, key is test_row_0/C:col10/1733399884723/Put/seqid=0 2024-12-05T11:58:06,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742192_1368 (size=12151) 2024-12-05T11:58:06,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/2e42290099d54627ba68c00f0db854f6 2024-12-05T11:58:06,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/f8300cc0a5ed4a4f8320816b6658f407 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f8300cc0a5ed4a4f8320816b6658f407 2024-12-05T11:58:06,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f8300cc0a5ed4a4f8320816b6658f407, entries=200, sequenceid=178, filesize=38.8 K 2024-12-05T11:58:06,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/0277ca04c61d449085d37dc6d89d2774 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/0277ca04c61d449085d37dc6d89d2774 2024-12-05T11:58:06,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/0277ca04c61d449085d37dc6d89d2774, entries=150, sequenceid=178, filesize=11.9 K 2024-12-05T11:58:06,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T11:58:06,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:06,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:06,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:06,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/2e42290099d54627ba68c00f0db854f6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/2e42290099d54627ba68c00f0db854f6 2024-12-05T11:58:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:06,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/2e42290099d54627ba68c00f0db854f6, entries=150, sequenceid=178, filesize=11.9 K 2024-12-05T11:58:06,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 6fb7227437f8ecf2f0f7089b8972c742 in 1034ms, sequenceid=178, compaction requested=true 2024-12-05T11:58:06,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:06,339 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:06,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:06,340 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:06,340 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/A is initiating minor compaction (all files) 2024-12-05T11:58:06,340 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/A in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,340 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/106a1c4fc148499b8469a5b8ed819320, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f1929850c9ab405bbdbb5c9cb04d3b33, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f8300cc0a5ed4a4f8320816b6658f407] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=99.9 K 2024-12-05T11:58:06,340 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,340 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/106a1c4fc148499b8469a5b8ed819320, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f1929850c9ab405bbdbb5c9cb04d3b33, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f8300cc0a5ed4a4f8320816b6658f407] 2024-12-05T11:58:06,341 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 106a1c4fc148499b8469a5b8ed819320, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733399881855 2024-12-05T11:58:06,341 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1929850c9ab405bbdbb5c9cb04d3b33, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733399882513 2024-12-05T11:58:06,341 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8300cc0a5ed4a4f8320816b6658f407, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733399884712 2024-12-05T11:58:06,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:06,342 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:06,343 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:06,343 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/B is initiating minor compaction (all files) 2024-12-05T11:58:06,343 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/B in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,343 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/3727766642f340c0ab928b274a5822b5, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f350b87479cf47a7836a813889486d2b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/0277ca04c61d449085d37dc6d89d2774] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=35.9 K 2024-12-05T11:58:06,345 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 3727766642f340c0ab928b274a5822b5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733399881855 2024-12-05T11:58:06,346 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting f350b87479cf47a7836a813889486d2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733399882513 2024-12-05T11:58:06,346 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0277ca04c61d449085d37dc6d89d2774, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733399884714 2024-12-05T11:58:06,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:06,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:06,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:06,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:06,355 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:06,360 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#B#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:06,361 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/6e262a49461044fca25edb015fdec0a3 is 50, key is test_row_0/B:col10/1733399884723/Put/seqid=0 2024-12-05T11:58:06,367 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205b5389e4b57ae499a8f02b22b70f31795_6fb7227437f8ecf2f0f7089b8972c742 store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:06,369 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205b5389e4b57ae499a8f02b22b70f31795_6fb7227437f8ecf2f0f7089b8972c742, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:06,369 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b5389e4b57ae499a8f02b22b70f31795_6fb7227437f8ecf2f0f7089b8972c742 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T11:58:06,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742193_1369 (size=12561) 2024-12-05T11:58:06,397 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/6e262a49461044fca25edb015fdec0a3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6e262a49461044fca25edb015fdec0a3 2024-12-05T11:58:06,415 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/B of 6fb7227437f8ecf2f0f7089b8972c742 into 6e262a49461044fca25edb015fdec0a3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:06,415 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:06,415 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/B, priority=13, startTime=1733399886341; duration=0sec 2024-12-05T11:58:06,415 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:06,415 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:B 2024-12-05T11:58:06,415 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:06,416 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:06,416 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/C is initiating minor compaction (all files) 2024-12-05T11:58:06,416 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/C in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,416 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/201f04cf91214e71975e1f127cc36d20, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/eea73ecd03804e62b1aa4636dcad6433, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/2e42290099d54627ba68c00f0db854f6] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=35.9 K 2024-12-05T11:58:06,417 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 201f04cf91214e71975e1f127cc36d20, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733399881855 2024-12-05T11:58:06,417 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting eea73ecd03804e62b1aa4636dcad6433, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733399882513 2024-12-05T11:58:06,417 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e42290099d54627ba68c00f0db854f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733399884714 2024-12-05T11:58:06,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742194_1370 (size=4469) 2024-12-05T11:58:06,448 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#C#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:06,449 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/d1680703f6904a72aa8cd866c0953680 is 50, key is test_row_0/C:col10/1733399884723/Put/seqid=0 2024-12-05T11:58:06,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742195_1371 (size=12561) 2024-12-05T11:58:06,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T11:58:06,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,485 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-05T11:58:06,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054570bea3511242d691b4e64431826c52_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399885359/Put/seqid=0 2024-12-05T11:58:06,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742196_1372 (size=12304) 2024-12-05T11:58:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:06,521 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054570bea3511242d691b4e64431826c52_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054570bea3511242d691b4e64431826c52_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:06,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/5c8a8949d7344d1bb7e5181de0c4baf9, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:06,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/5c8a8949d7344d1bb7e5181de0c4baf9 is 175, key is test_row_0/A:col10/1733399885359/Put/seqid=0 2024-12-05T11:58:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:06,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742197_1373 (size=31105) 2024-12-05T11:58:06,552 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/5c8a8949d7344d1bb7e5181de0c4baf9 2024-12-05T11:58:06,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/a7746edf0d394464b7cfdc3bf2c8e957 is 50, key is test_row_0/B:col10/1733399885359/Put/seqid=0 2024-12-05T11:58:06,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742198_1374 (size=12151) 2024-12-05T11:58:06,577 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/a7746edf0d394464b7cfdc3bf2c8e957 2024-12-05T11:58:06,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/f81c263c76c249779ba240cfe380f4bd is 50, key is test_row_0/C:col10/1733399885359/Put/seqid=0 2024-12-05T11:58:06,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742199_1375 (size=12151) 2024-12-05T11:58:06,591 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/f81c263c76c249779ba240cfe380f4bd 2024-12-05T11:58:06,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/5c8a8949d7344d1bb7e5181de0c4baf9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/5c8a8949d7344d1bb7e5181de0c4baf9 2024-12-05T11:58:06,602 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/5c8a8949d7344d1bb7e5181de0c4baf9, entries=150, sequenceid=195, filesize=30.4 K 2024-12-05T11:58:06,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/a7746edf0d394464b7cfdc3bf2c8e957 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/a7746edf0d394464b7cfdc3bf2c8e957 2024-12-05T11:58:06,610 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/a7746edf0d394464b7cfdc3bf2c8e957, entries=150, sequenceid=195, filesize=11.9 K 2024-12-05T11:58:06,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/f81c263c76c249779ba240cfe380f4bd as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/f81c263c76c249779ba240cfe380f4bd 2024-12-05T11:58:06,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399946612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:06,628 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/f81c263c76c249779ba240cfe380f4bd, entries=150, sequenceid=195, filesize=11.9 K 2024-12-05T11:58:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:06,633 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 6fb7227437f8ecf2f0f7089b8972c742 in 148ms, sequenceid=195, compaction requested=false 2024-12-05T11:58:06,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:06,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-05T11:58:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-05T11:58:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:06,634 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:06,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-05T11:58:06,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-05T11:58:06,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3840 sec 2024-12-05T11:58:06,643 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.3920 sec 2024-12-05T11:58:06,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120540c8cc2e4bc24342a8e9b17f0440ef6b_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399886629/Put/seqid=0 2024-12-05T11:58:06,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742200_1376 (size=17284) 2024-12-05T11:58:06,700 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:06,706 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120540c8cc2e4bc24342a8e9b17f0440ef6b_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120540c8cc2e4bc24342a8e9b17f0440ef6b_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:06,709 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/c515192b0ddd4d559f03e56acebaa7c3, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:06,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/c515192b0ddd4d559f03e56acebaa7c3 is 175, key is test_row_0/A:col10/1733399886629/Put/seqid=0 2024-12-05T11:58:06,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399946713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:06,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399946729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:06,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742201_1377 (size=48389) 2024-12-05T11:58:06,744 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/c515192b0ddd4d559f03e56acebaa7c3 2024-12-05T11:58:06,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399946738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:06,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/d21a0f1259f64e618b8a81d47016623d is 50, key is test_row_0/B:col10/1733399886629/Put/seqid=0 2024-12-05T11:58:06,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742202_1378 (size=12151) 2024-12-05T11:58:06,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/d21a0f1259f64e618b8a81d47016623d 2024-12-05T11:58:06,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/d8d9511cee5440368291922e9e270623 is 50, key is test_row_0/C:col10/1733399886629/Put/seqid=0 2024-12-05T11:58:06,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742203_1379 (size=12151) 2024-12-05T11:58:06,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/d8d9511cee5440368291922e9e270623 2024-12-05T11:58:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/c515192b0ddd4d559f03e56acebaa7c3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c515192b0ddd4d559f03e56acebaa7c3 2024-12-05T11:58:06,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c515192b0ddd4d559f03e56acebaa7c3, entries=250, sequenceid=217, filesize=47.3 K 2024-12-05T11:58:06,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/d21a0f1259f64e618b8a81d47016623d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d21a0f1259f64e618b8a81d47016623d 2024-12-05T11:58:06,836 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#A#compaction#312 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:06,837 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/9561681e8e1c41c49e9c25d8d914854c is 175, key is test_row_0/A:col10/1733399884723/Put/seqid=0 2024-12-05T11:58:06,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d21a0f1259f64e618b8a81d47016623d, entries=150, sequenceid=217, filesize=11.9 K 2024-12-05T11:58:06,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/d8d9511cee5440368291922e9e270623 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d8d9511cee5440368291922e9e270623 2024-12-05T11:58:06,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399946839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:06,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d8d9511cee5440368291922e9e270623, entries=150, sequenceid=217, filesize=11.9 K 2024-12-05T11:58:06,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 6fb7227437f8ecf2f0f7089b8972c742 in 220ms, sequenceid=217, compaction requested=true 2024-12-05T11:58:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:A, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:B, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T11:58:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-05T11:58:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:06,858 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:06,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-05T11:58:06,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742204_1380 (size=31515) 2024-12-05T11:58:06,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051de2279332b4419abcb87c6a54a8e8aa_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399886856/Put/seqid=0 2024-12-05T11:58:06,881 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/d1680703f6904a72aa8cd866c0953680 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d1680703f6904a72aa8cd866c0953680 2024-12-05T11:58:06,889 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/C of 6fb7227437f8ecf2f0f7089b8972c742 into d1680703f6904a72aa8cd866c0953680(size=12.3 K), total size for store is 36.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:06,889 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:06,889 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/C, priority=13, startTime=1733399886346; duration=0sec 2024-12-05T11:58:06,889 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-05T11:58:06,889 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:C 2024-12-05T11:58:06,889 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-12-05T11:58:06,890 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-05T11:58:06,890 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-05T11:58:06,891 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. because compaction request was cancelled 2024-12-05T11:58:06,891 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:A 2024-12-05T11:58:06,891 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:C 2024-12-05T11:58:06,891 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:06,892 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:06,892 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/B is initiating minor compaction (all files) 2024-12-05T11:58:06,892 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/B in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:06,892 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6e262a49461044fca25edb015fdec0a3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/a7746edf0d394464b7cfdc3bf2c8e957, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d21a0f1259f64e618b8a81d47016623d] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=36.0 K 2024-12-05T11:58:06,892 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e262a49461044fca25edb015fdec0a3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733399884714 2024-12-05T11:58:06,892 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting a7746edf0d394464b7cfdc3bf2c8e957, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733399885359 2024-12-05T11:58:06,893 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d21a0f1259f64e618b8a81d47016623d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733399886593 2024-12-05T11:58:06,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742205_1381 (size=14794) 2024-12-05T11:58:06,905 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#B#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:06,905 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/57bc404d80a140929db63a1bbb6d5e5d is 50, key is test_row_0/B:col10/1733399886629/Put/seqid=0 2024-12-05T11:58:06,906 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:06,910 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051de2279332b4419abcb87c6a54a8e8aa_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051de2279332b4419abcb87c6a54a8e8aa_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:06,911 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/af45c23e4a9647bd87a734398c4970bd, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:06,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/af45c23e4a9647bd87a734398c4970bd is 175, key is test_row_0/A:col10/1733399886856/Put/seqid=0 2024-12-05T11:58:06,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742206_1382 (size=12663) 2024-12-05T11:58:06,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742207_1383 (size=39749) 2024-12-05T11:58:06,925 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/af45c23e4a9647bd87a734398c4970bd 2024-12-05T11:58:06,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/bc4ee71f1cc1425fb0a5f80327e2f57b is 50, key is test_row_0/B:col10/1733399886856/Put/seqid=0 2024-12-05T11:58:06,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742208_1384 (size=12151) 2024-12-05T11:58:06,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/bc4ee71f1cc1425fb0a5f80327e2f57b 2024-12-05T11:58:06,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399946959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:06,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399946961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:06,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/cabc1199328945fc83896b03ddc2f3fc is 50, key is test_row_0/C:col10/1733399886856/Put/seqid=0 2024-12-05T11:58:06,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399946972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:06,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399946975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742209_1385 (size=12151) 2024-12-05T11:58:07,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/cabc1199328945fc83896b03ddc2f3fc 2024-12-05T11:58:07,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/af45c23e4a9647bd87a734398c4970bd as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/af45c23e4a9647bd87a734398c4970bd 2024-12-05T11:58:07,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/af45c23e4a9647bd87a734398c4970bd, entries=200, sequenceid=232, filesize=38.8 K 2024-12-05T11:58:07,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/bc4ee71f1cc1425fb0a5f80327e2f57b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/bc4ee71f1cc1425fb0a5f80327e2f57b 2024-12-05T11:58:07,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/bc4ee71f1cc1425fb0a5f80327e2f57b, entries=150, sequenceid=232, filesize=11.9 K 2024-12-05T11:58:07,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/cabc1199328945fc83896b03ddc2f3fc as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/cabc1199328945fc83896b03ddc2f3fc 2024-12-05T11:58:07,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/cabc1199328945fc83896b03ddc2f3fc, entries=150, sequenceid=232, filesize=11.9 K 2024-12-05T11:58:07,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6fb7227437f8ecf2f0f7089b8972c742 in 185ms, sequenceid=232, compaction requested=true 2024-12-05T11:58:07,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:07,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:A, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:07,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T11:58:07,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:07,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-05T11:58:07,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:07,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-05T11:58:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:07,064 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:07,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-05T11:58:07,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ec69c4d099594914a2057381fe0726ca_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399886970/Put/seqid=0 2024-12-05T11:58:07,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742210_1386 (size=14794) 2024-12-05T11:58:07,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399947103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399947104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399947104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399947112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399947112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399947227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399947228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399947228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399947228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399947228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,271 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/9561681e8e1c41c49e9c25d8d914854c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9561681e8e1c41c49e9c25d8d914854c 2024-12-05T11:58:07,278 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/A of 6fb7227437f8ecf2f0f7089b8972c742 into 9561681e8e1c41c49e9c25d8d914854c(size=30.8 K), total size for store is 147.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:07,278 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:07,278 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/A, priority=13, startTime=1733399886339; duration=0sec 2024-12-05T11:58:07,278 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-05T11:58:07,278 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:A 2024-12-05T11:58:07,278 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:A 2024-12-05T11:58:07,279 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:07,280 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:07,280 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/C is initiating minor compaction (all files) 2024-12-05T11:58:07,280 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/C in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:07,280 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d1680703f6904a72aa8cd866c0953680, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/f81c263c76c249779ba240cfe380f4bd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d8d9511cee5440368291922e9e270623, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/cabc1199328945fc83896b03ddc2f3fc] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=47.9 K 2024-12-05T11:58:07,281 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1680703f6904a72aa8cd866c0953680, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733399884714 2024-12-05T11:58:07,287 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f81c263c76c249779ba240cfe380f4bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733399885359 2024-12-05T11:58:07,288 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8d9511cee5440368291922e9e270623, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733399886593 2024-12-05T11:58:07,288 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting cabc1199328945fc83896b03ddc2f3fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733399886707 2024-12-05T11:58:07,309 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#C#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:07,309 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/8bcdb0531cb34a218ad6b918a2476342 is 50, key is test_row_0/C:col10/1733399886856/Put/seqid=0 2024-12-05T11:58:07,328 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/57bc404d80a140929db63a1bbb6d5e5d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/57bc404d80a140929db63a1bbb6d5e5d 2024-12-05T11:58:07,334 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/B of 6fb7227437f8ecf2f0f7089b8972c742 into 57bc404d80a140929db63a1bbb6d5e5d(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:07,334 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:07,334 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/B, priority=13, startTime=1733399886854; duration=0sec 2024-12-05T11:58:07,334 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-05T11:58:07,334 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:B 2024-12-05T11:58:07,334 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:07,336 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150758 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:07,336 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/A is initiating minor compaction (all files) 2024-12-05T11:58:07,336 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/A in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:07,336 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9561681e8e1c41c49e9c25d8d914854c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/5c8a8949d7344d1bb7e5181de0c4baf9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c515192b0ddd4d559f03e56acebaa7c3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/af45c23e4a9647bd87a734398c4970bd] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=147.2 K 2024-12-05T11:58:07,336 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:07,336 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9561681e8e1c41c49e9c25d8d914854c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/5c8a8949d7344d1bb7e5181de0c4baf9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c515192b0ddd4d559f03e56acebaa7c3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/af45c23e4a9647bd87a734398c4970bd] 2024-12-05T11:58:07,337 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9561681e8e1c41c49e9c25d8d914854c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733399884714 2024-12-05T11:58:07,337 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c8a8949d7344d1bb7e5181de0c4baf9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733399885359 2024-12-05T11:58:07,337 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c515192b0ddd4d559f03e56acebaa7c3, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733399886593 2024-12-05T11:58:07,338 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting af45c23e4a9647bd87a734398c4970bd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733399886707 2024-12-05T11:58:07,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742211_1387 (size=12697) 2024-12-05T11:58:07,355 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:07,360 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120579c92a2434c14b18ace150b4b0ee8bd5_6fb7227437f8ecf2f0f7089b8972c742 store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:07,362 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120579c92a2434c14b18ace150b4b0ee8bd5_6fb7227437f8ecf2f0f7089b8972c742, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:07,362 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120579c92a2434c14b18ace150b4b0ee8bd5_6fb7227437f8ecf2f0f7089b8972c742 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:07,371 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/8bcdb0531cb34a218ad6b918a2476342 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/8bcdb0531cb34a218ad6b918a2476342 2024-12-05T11:58:07,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742212_1388 (size=4469) 2024-12-05T11:58:07,377 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/C of 6fb7227437f8ecf2f0f7089b8972c742 into 8bcdb0531cb34a218ad6b918a2476342(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:07,377 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:07,377 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/C, priority=12, startTime=1733399887044; duration=0sec 2024-12-05T11:58:07,377 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T11:58:07,377 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:C 2024-12-05T11:58:07,377 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:B 2024-12-05T11:58:07,377 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:C 2024-12-05T11:58:07,377 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-05T11:58:07,377 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-05T11:58:07,377 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-05T11:58:07,377 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. because compaction request was cancelled 2024-12-05T11:58:07,377 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:B 2024-12-05T11:58:07,377 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-05T11:58:07,378 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-05T11:58:07,378 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-05T11:58:07,378 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. because compaction request was cancelled 2024-12-05T11:58:07,378 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:C 2024-12-05T11:58:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T11:58:07,399 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:07,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-05T11:58:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T11:58:07,404 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:07,405 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:07,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:07,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399947438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399947438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399947438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399947447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399947447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,496 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:07,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ec69c4d099594914a2057381fe0726ca_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ec69c4d099594914a2057381fe0726ca_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:07,500 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/bb5e884a30ea4705b283cd004b95167e, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:07,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/bb5e884a30ea4705b283cd004b95167e is 175, key is test_row_0/A:col10/1733399886970/Put/seqid=0 2024-12-05T11:58:07,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742213_1389 (size=39749) 2024-12-05T11:58:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T11:58:07,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T11:58:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:07,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T11:58:07,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T11:58:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:07,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:07,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399947748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399947758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399947758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399947758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399947758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:07,776 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:07,776 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/c633a05d6d984262b9df9b4285c87b0a is 175, key is test_row_0/A:col10/1733399886856/Put/seqid=0 2024-12-05T11:58:07,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742214_1390 (size=31651) 2024-12-05T11:58:07,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T11:58:07,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:07,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:07,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:07,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:07,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:07,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/bb5e884a30ea4705b283cd004b95167e 2024-12-05T11:58:07,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/f352ba5d87ab45f1a0d6251abff1e7ef is 50, key is test_row_0/B:col10/1733399886970/Put/seqid=0 2024-12-05T11:58:07,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742215_1391 (size=12151) 2024-12-05T11:58:07,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/f352ba5d87ab45f1a0d6251abff1e7ef 2024-12-05T11:58:07,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/f0100fdfb27e4a5081cd3072330d2ca9 is 50, key is test_row_0/C:col10/1733399886970/Put/seqid=0 2024-12-05T11:58:07,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742216_1392 (size=12151) 2024-12-05T11:58:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T11:58:08,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T11:58:08,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:08,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:08,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:08,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:08,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:08,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:08,035 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T11:58:08,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T11:58:08,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:08,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:08,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:08,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:08,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:08,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:08,189 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/c633a05d6d984262b9df9b4285c87b0a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c633a05d6d984262b9df9b4285c87b0a 2024-12-05T11:58:08,193 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/A of 6fb7227437f8ecf2f0f7089b8972c742 into c633a05d6d984262b9df9b4285c87b0a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:08,193 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:08,193 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/A, priority=12, startTime=1733399887044; duration=0sec 2024-12-05T11:58:08,193 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:08,193 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:A 2024-12-05T11:58:08,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399948268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:08,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399948278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:08,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399948278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:08,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399948278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:08,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399948278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:08,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T11:58:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:08,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:08,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/f0100fdfb27e4a5081cd3072330d2ca9 2024-12-05T11:58:08,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/bb5e884a30ea4705b283cd004b95167e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/bb5e884a30ea4705b283cd004b95167e 2024-12-05T11:58:08,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/bb5e884a30ea4705b283cd004b95167e, entries=200, sequenceid=255, filesize=38.8 K 2024-12-05T11:58:08,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/f352ba5d87ab45f1a0d6251abff1e7ef as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f352ba5d87ab45f1a0d6251abff1e7ef 2024-12-05T11:58:08,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f352ba5d87ab45f1a0d6251abff1e7ef, entries=150, sequenceid=255, filesize=11.9 K 2024-12-05T11:58:08,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/f0100fdfb27e4a5081cd3072330d2ca9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/f0100fdfb27e4a5081cd3072330d2ca9 2024-12-05T11:58:08,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/f0100fdfb27e4a5081cd3072330d2ca9, entries=150, sequenceid=255, filesize=11.9 K 2024-12-05T11:58:08,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 6fb7227437f8ecf2f0f7089b8972c742 in 1355ms, sequenceid=255, compaction requested=true 2024-12-05T11:58:08,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:08,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:08,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:08,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:08,420 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-05T11:58:08,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:08,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:08,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:08,420 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:08,420 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-05T11:58:08,420 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-05T11:58:08,420 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. because compaction request was cancelled 2024-12-05T11:58:08,420 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:A 2024-12-05T11:58:08,420 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-05T11:58:08,421 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:08,421 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-05T11:58:08,421 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/B is initiating minor compaction (all files) 2024-12-05T11:58:08,421 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-05T11:58:08,421 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. because compaction request was cancelled 2024-12-05T11:58:08,421 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/B in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:08,421 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:C 2024-12-05T11:58:08,421 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/57bc404d80a140929db63a1bbb6d5e5d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/bc4ee71f1cc1425fb0a5f80327e2f57b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f352ba5d87ab45f1a0d6251abff1e7ef] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=36.1 K 2024-12-05T11:58:08,421 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 57bc404d80a140929db63a1bbb6d5e5d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733399886593 2024-12-05T11:58:08,421 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting bc4ee71f1cc1425fb0a5f80327e2f57b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733399886707 2024-12-05T11:58:08,422 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting f352ba5d87ab45f1a0d6251abff1e7ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733399886970 2024-12-05T11:58:08,430 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:08,431 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/89c55fafc47b4557ae301c8f10bdcedd is 50, key is test_row_0/B:col10/1733399886970/Put/seqid=0 2024-12-05T11:58:08,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742217_1393 (size=12765) 2024-12-05T11:58:08,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T11:58:08,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:08,488 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T11:58:08,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051c7ad7a3052847e294d71c712b47b2b8_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399887103/Put/seqid=0 2024-12-05T11:58:08,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742218_1394 (size=12454) 2024-12-05T11:58:08,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:08,511 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051c7ad7a3052847e294d71c712b47b2b8_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051c7ad7a3052847e294d71c712b47b2b8_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:08,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/fd1e49a39a5a4f49b450121313206f4f, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:08,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/fd1e49a39a5a4f49b450121313206f4f is 175, key is test_row_0/A:col10/1733399887103/Put/seqid=0 2024-12-05T11:58:08,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742219_1395 (size=31255) 2024-12-05T11:58:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T11:58:08,841 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/89c55fafc47b4557ae301c8f10bdcedd as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/89c55fafc47b4557ae301c8f10bdcedd 2024-12-05T11:58:08,845 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/B of 6fb7227437f8ecf2f0f7089b8972c742 into 89c55fafc47b4557ae301c8f10bdcedd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:08,846 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:08,846 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/B, priority=13, startTime=1733399888420; duration=0sec 2024-12-05T11:58:08,846 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:08,846 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:B 2024-12-05T11:58:08,925 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/fd1e49a39a5a4f49b450121313206f4f 2024-12-05T11:58:08,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/83b0bb7a4441457a89a0d293a18ebe23 is 50, key is test_row_0/B:col10/1733399887103/Put/seqid=0 2024-12-05T11:58:08,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742220_1396 (size=12301) 2024-12-05T11:58:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:09,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:09,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399949314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399949317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399949318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399949319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399949319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,340 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/83b0bb7a4441457a89a0d293a18ebe23 2024-12-05T11:58:09,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/6cb4856d7fa14aa2aec3bf4695cb1dc3 is 50, key is test_row_0/C:col10/1733399887103/Put/seqid=0 2024-12-05T11:58:09,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742221_1397 (size=12301) 2024-12-05T11:58:09,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399949427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399949427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399949427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399949437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399949438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T11:58:09,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399949637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399949647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399949647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399949648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399949648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,749 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/6cb4856d7fa14aa2aec3bf4695cb1dc3 2024-12-05T11:58:09,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/fd1e49a39a5a4f49b450121313206f4f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/fd1e49a39a5a4f49b450121313206f4f 2024-12-05T11:58:09,755 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/fd1e49a39a5a4f49b450121313206f4f, entries=150, sequenceid=273, filesize=30.5 K 2024-12-05T11:58:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/83b0bb7a4441457a89a0d293a18ebe23 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/83b0bb7a4441457a89a0d293a18ebe23 2024-12-05T11:58:09,759 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/83b0bb7a4441457a89a0d293a18ebe23, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T11:58:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/6cb4856d7fa14aa2aec3bf4695cb1dc3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/6cb4856d7fa14aa2aec3bf4695cb1dc3 2024-12-05T11:58:09,763 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/6cb4856d7fa14aa2aec3bf4695cb1dc3, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T11:58:09,763 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6fb7227437f8ecf2f0f7089b8972c742 in 1275ms, sequenceid=273, compaction requested=true 2024-12-05T11:58:09,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:09,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:09,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-05T11:58:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-05T11:58:09,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-05T11:58:09,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3590 sec 2024-12-05T11:58:09,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.3670 sec 2024-12-05T11:58:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:09,951 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:09,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T11:58:09,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205744ea6d5c91f4da5a0538d6ff815c5ae_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399889318/Put/seqid=0 2024-12-05T11:58:09,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742222_1398 (size=14994) 2024-12-05T11:58:09,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399949965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399949967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399949967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399949971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:09,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399949972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399950077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399950078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399950078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399950078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399950088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399950287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399950287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399950288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399950288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399950298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,361 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:10,364 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205744ea6d5c91f4da5a0538d6ff815c5ae_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205744ea6d5c91f4da5a0538d6ff815c5ae_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:10,365 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/9a035edaee034f60acd99897f7055bd0, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:10,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/9a035edaee034f60acd99897f7055bd0 is 175, key is test_row_0/A:col10/1733399889318/Put/seqid=0 2024-12-05T11:58:10,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742223_1399 (size=39949) 2024-12-05T11:58:10,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399950598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399950598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399950598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399950598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399950608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:10,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:10,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:10,768 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/9a035edaee034f60acd99897f7055bd0 2024-12-05T11:58:10,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/1db12eb5a34947358c2364c9850cdf03 is 50, key is test_row_0/B:col10/1733399889318/Put/seqid=0 2024-12-05T11:58:10,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742224_1400 (size=12301) 2024-12-05T11:58:10,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/1db12eb5a34947358c2364c9850cdf03 2024-12-05T11:58:10,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/66d18cb6fb4d4ff6a2798a52251fd967 is 50, key is test_row_0/C:col10/1733399889318/Put/seqid=0 2024-12-05T11:58:10,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742225_1401 (size=12301) 2024-12-05T11:58:11,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399951108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:11,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399951108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:11,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:11,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399951108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:11,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:11,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:11,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:11,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:11,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399951118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:11,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399951118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:11,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/66d18cb6fb4d4ff6a2798a52251fd967 2024-12-05T11:58:11,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/9a035edaee034f60acd99897f7055bd0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9a035edaee034f60acd99897f7055bd0 2024-12-05T11:58:11,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9a035edaee034f60acd99897f7055bd0, entries=200, sequenceid=297, filesize=39.0 K 2024-12-05T11:58:11,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/1db12eb5a34947358c2364c9850cdf03 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/1db12eb5a34947358c2364c9850cdf03 2024-12-05T11:58:11,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/1db12eb5a34947358c2364c9850cdf03, entries=150, sequenceid=297, filesize=12.0 K 2024-12-05T11:58:11,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/66d18cb6fb4d4ff6a2798a52251fd967 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/66d18cb6fb4d4ff6a2798a52251fd967 2024-12-05T11:58:11,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/66d18cb6fb4d4ff6a2798a52251fd967, entries=150, sequenceid=297, filesize=12.0 K 2024-12-05T11:58:11,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 6fb7227437f8ecf2f0f7089b8972c742 in 1248ms, sequenceid=297, compaction requested=true 2024-12-05T11:58:11,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:11,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:11,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:11,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:11,199 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:11,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:11,199 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:11,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:11,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:11,200 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:11,200 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/B is initiating minor compaction (all files) 2024-12-05T11:58:11,200 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142604 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:11,200 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/A is initiating minor compaction (all files) 2024-12-05T11:58:11,200 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/B in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:11,200 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/A in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:11,200 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c633a05d6d984262b9df9b4285c87b0a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/bb5e884a30ea4705b283cd004b95167e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/fd1e49a39a5a4f49b450121313206f4f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9a035edaee034f60acd99897f7055bd0] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=139.3 K 2024-12-05T11:58:11,200 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/89c55fafc47b4557ae301c8f10bdcedd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/83b0bb7a4441457a89a0d293a18ebe23, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/1db12eb5a34947358c2364c9850cdf03] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=36.5 K 2024-12-05T11:58:11,200 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:11,200 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c633a05d6d984262b9df9b4285c87b0a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/bb5e884a30ea4705b283cd004b95167e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/fd1e49a39a5a4f49b450121313206f4f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9a035edaee034f60acd99897f7055bd0] 2024-12-05T11:58:11,201 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 89c55fafc47b4557ae301c8f10bdcedd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733399886970 2024-12-05T11:58:11,201 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c633a05d6d984262b9df9b4285c87b0a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733399886707 2024-12-05T11:58:11,201 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 83b0bb7a4441457a89a0d293a18ebe23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733399887098 2024-12-05T11:58:11,201 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1db12eb5a34947358c2364c9850cdf03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733399889309 2024-12-05T11:58:11,201 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb5e884a30ea4705b283cd004b95167e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733399886959 2024-12-05T11:58:11,201 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd1e49a39a5a4f49b450121313206f4f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733399887098 2024-12-05T11:58:11,201 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a035edaee034f60acd99897f7055bd0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733399889309 2024-12-05T11:58:11,207 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#B#compaction#337 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:11,207 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:11,207 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/dabbe08d35c1496abc0aa7ff1688238a is 50, key is test_row_0/B:col10/1733399889318/Put/seqid=0 2024-12-05T11:58:11,209 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120595d905e78ecc40c09c20ed091301f54a_6fb7227437f8ecf2f0f7089b8972c742 store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:11,212 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120595d905e78ecc40c09c20ed091301f54a_6fb7227437f8ecf2f0f7089b8972c742, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:11,212 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120595d905e78ecc40c09c20ed091301f54a_6fb7227437f8ecf2f0f7089b8972c742 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:11,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742226_1402 (size=13017) 2024-12-05T11:58:11,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742227_1403 (size=4469) 2024-12-05T11:58:11,218 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#A#compaction#338 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:11,218 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/1dbe95545354482d85b8b455675db494 is 175, key is test_row_0/A:col10/1733399889318/Put/seqid=0 2024-12-05T11:58:11,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742228_1404 (size=31937) 2024-12-05T11:58:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T11:58:11,568 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:11,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-05T11:58:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T11:58:11,570 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:11,570 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:11,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:11,624 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/dabbe08d35c1496abc0aa7ff1688238a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/dabbe08d35c1496abc0aa7ff1688238a 2024-12-05T11:58:11,628 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/1dbe95545354482d85b8b455675db494 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/1dbe95545354482d85b8b455675db494 2024-12-05T11:58:11,632 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/A of 6fb7227437f8ecf2f0f7089b8972c742 into 1dbe95545354482d85b8b455675db494(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:11,632 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:11,632 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/A, priority=12, startTime=1733399891199; duration=0sec 2024-12-05T11:58:11,632 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:11,632 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:A 2024-12-05T11:58:11,632 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:11,633 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:11,633 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/C is initiating minor compaction (all files) 2024-12-05T11:58:11,633 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/C in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:11,633 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/8bcdb0531cb34a218ad6b918a2476342, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/f0100fdfb27e4a5081cd3072330d2ca9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/6cb4856d7fa14aa2aec3bf4695cb1dc3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/66d18cb6fb4d4ff6a2798a52251fd967] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=48.3 K 2024-12-05T11:58:11,633 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bcdb0531cb34a218ad6b918a2476342, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733399886707 2024-12-05T11:58:11,633 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0100fdfb27e4a5081cd3072330d2ca9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733399886970 2024-12-05T11:58:11,633 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cb4856d7fa14aa2aec3bf4695cb1dc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733399887098 2024-12-05T11:58:11,634 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66d18cb6fb4d4ff6a2798a52251fd967, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733399889309 2024-12-05T11:58:11,635 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/B of 6fb7227437f8ecf2f0f7089b8972c742 into dabbe08d35c1496abc0aa7ff1688238a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:11,635 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:11,635 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/B, priority=13, startTime=1733399891199; duration=0sec 2024-12-05T11:58:11,635 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:11,635 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:B 2024-12-05T11:58:11,645 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#C#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:58:11,646 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/d6af5e2d6a4747fb9d0f83401f1077d7 is 50, key is test_row_0/C:col10/1733399889318/Put/seqid=0 2024-12-05T11:58:11,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742229_1405 (size=12983) 2024-12-05T11:58:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T11:58:11,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-05T11:58:11,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:11,723 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T11:58:11,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051dd5273e58e7493698cbff4659aea25e_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399889970/Put/seqid=0 2024-12-05T11:58:11,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742230_1406 (size=12454) 2024-12-05T11:58:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T11:58:12,066 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/d6af5e2d6a4747fb9d0f83401f1077d7 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d6af5e2d6a4747fb9d0f83401f1077d7 2024-12-05T11:58:12,072 DEBUG [regionserver/80666a11a09f:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-12-05T11:58:12,074 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/C of 6fb7227437f8ecf2f0f7089b8972c742 into d6af5e2d6a4747fb9d0f83401f1077d7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:12,074 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:12,074 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/C, priority=12, startTime=1733399891199; duration=0sec 2024-12-05T11:58:12,074 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:12,074 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:C 2024-12-05T11:58:12,076 WARN [regionserver/80666a11a09f:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-12-05T11:58:12,076 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9f202461f653437e9000dd71bd8a90fb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/7fd75e0d771440a88d1a9666340a239c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/2358756ad04a48938e43a07184f45419, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/960b053d219444c8a579b6c8fccd1ce6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/57c3c867711f4669a74c96bd06ff3ccb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/ca09db1599124239b45a5855819aeb04, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/82138d561955465d962e5e8f0b4ac797, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/cdec014816df4acdb629fe9683cd6f35, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/729520e0a9214037b854d3bd9fc5f4b5, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/106a1c4fc148499b8469a5b8ed819320, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f1929850c9ab405bbdbb5c9cb04d3b33, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f8300cc0a5ed4a4f8320816b6658f407, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9561681e8e1c41c49e9c25d8d914854c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/5c8a8949d7344d1bb7e5181de0c4baf9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c515192b0ddd4d559f03e56acebaa7c3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/af45c23e4a9647bd87a734398c4970bd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c633a05d6d984262b9df9b4285c87b0a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/bb5e884a30ea4705b283cd004b95167e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/fd1e49a39a5a4f49b450121313206f4f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9a035edaee034f60acd99897f7055bd0] to archive 2024-12-05T11:58:12,078 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:58:12,080 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9f202461f653437e9000dd71bd8a90fb to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9f202461f653437e9000dd71bd8a90fb 2024-12-05T11:58:12,081 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/7fd75e0d771440a88d1a9666340a239c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/7fd75e0d771440a88d1a9666340a239c 2024-12-05T11:58:12,082 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/2358756ad04a48938e43a07184f45419 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/2358756ad04a48938e43a07184f45419 2024-12-05T11:58:12,088 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/960b053d219444c8a579b6c8fccd1ce6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/960b053d219444c8a579b6c8fccd1ce6 2024-12-05T11:58:12,091 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/57c3c867711f4669a74c96bd06ff3ccb to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/57c3c867711f4669a74c96bd06ff3ccb 2024-12-05T11:58:12,092 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/ca09db1599124239b45a5855819aeb04 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/ca09db1599124239b45a5855819aeb04 2024-12-05T11:58:12,093 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/82138d561955465d962e5e8f0b4ac797 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/82138d561955465d962e5e8f0b4ac797 2024-12-05T11:58:12,096 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/cdec014816df4acdb629fe9683cd6f35 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/cdec014816df4acdb629fe9683cd6f35 2024-12-05T11:58:12,098 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/729520e0a9214037b854d3bd9fc5f4b5 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/729520e0a9214037b854d3bd9fc5f4b5 2024-12-05T11:58:12,099 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/106a1c4fc148499b8469a5b8ed819320 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/106a1c4fc148499b8469a5b8ed819320 2024-12-05T11:58:12,102 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f1929850c9ab405bbdbb5c9cb04d3b33 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f1929850c9ab405bbdbb5c9cb04d3b33 2024-12-05T11:58:12,103 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f8300cc0a5ed4a4f8320816b6658f407 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/f8300cc0a5ed4a4f8320816b6658f407 2024-12-05T11:58:12,106 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9561681e8e1c41c49e9c25d8d914854c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9561681e8e1c41c49e9c25d8d914854c 2024-12-05T11:58:12,107 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/5c8a8949d7344d1bb7e5181de0c4baf9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/5c8a8949d7344d1bb7e5181de0c4baf9 2024-12-05T11:58:12,108 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c515192b0ddd4d559f03e56acebaa7c3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c515192b0ddd4d559f03e56acebaa7c3 2024-12-05T11:58:12,109 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/af45c23e4a9647bd87a734398c4970bd to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/af45c23e4a9647bd87a734398c4970bd 2024-12-05T11:58:12,110 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c633a05d6d984262b9df9b4285c87b0a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c633a05d6d984262b9df9b4285c87b0a 2024-12-05T11:58:12,111 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/bb5e884a30ea4705b283cd004b95167e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/bb5e884a30ea4705b283cd004b95167e 2024-12-05T11:58:12,112 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/fd1e49a39a5a4f49b450121313206f4f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/fd1e49a39a5a4f49b450121313206f4f 2024-12-05T11:58:12,113 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9a035edaee034f60acd99897f7055bd0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/9a035edaee034f60acd99897f7055bd0 2024-12-05T11:58:12,116 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/c7b298b322b4459a9ce5f954aeb105c7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/5029606044184201ae563322ecdbd5fa, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/0da966a1fe9c4a1baa81443100cf2920, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d6bd84fbf16b4e978f122578b1760d13, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/e85095716a1a49aeb1777eafee423ab2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/27c26b83649345ac96c5105f574def4b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6d8d723db96d466698e16d210dd3ad2c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/ec53497457f342d08367031d8f40edb1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/3727766642f340c0ab928b274a5822b5, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d14b498445d04bdb854fd29eb6cbd682, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f350b87479cf47a7836a813889486d2b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6e262a49461044fca25edb015fdec0a3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/0277ca04c61d449085d37dc6d89d2774, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/a7746edf0d394464b7cfdc3bf2c8e957, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/57bc404d80a140929db63a1bbb6d5e5d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d21a0f1259f64e618b8a81d47016623d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/bc4ee71f1cc1425fb0a5f80327e2f57b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/89c55fafc47b4557ae301c8f10bdcedd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f352ba5d87ab45f1a0d6251abff1e7ef, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/83b0bb7a4441457a89a0d293a18ebe23, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/1db12eb5a34947358c2364c9850cdf03] to archive 2024-12-05T11:58:12,117 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:58:12,119 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/c7b298b322b4459a9ce5f954aeb105c7 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/c7b298b322b4459a9ce5f954aeb105c7 2024-12-05T11:58:12,120 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/5029606044184201ae563322ecdbd5fa to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/5029606044184201ae563322ecdbd5fa 2024-12-05T11:58:12,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:12,122 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/0da966a1fe9c4a1baa81443100cf2920 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/0da966a1fe9c4a1baa81443100cf2920 2024-12-05T11:58:12,124 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d6bd84fbf16b4e978f122578b1760d13 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d6bd84fbf16b4e978f122578b1760d13 2024-12-05T11:58:12,125 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/e85095716a1a49aeb1777eafee423ab2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/e85095716a1a49aeb1777eafee423ab2 2024-12-05T11:58:12,128 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/27c26b83649345ac96c5105f574def4b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/27c26b83649345ac96c5105f574def4b 2024-12-05T11:58:12,129 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6d8d723db96d466698e16d210dd3ad2c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6d8d723db96d466698e16d210dd3ad2c 2024-12-05T11:58:12,129 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/ec53497457f342d08367031d8f40edb1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/ec53497457f342d08367031d8f40edb1 2024-12-05T11:58:12,130 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/3727766642f340c0ab928b274a5822b5 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/3727766642f340c0ab928b274a5822b5 2024-12-05T11:58:12,131 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d14b498445d04bdb854fd29eb6cbd682 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d14b498445d04bdb854fd29eb6cbd682 2024-12-05T11:58:12,133 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f350b87479cf47a7836a813889486d2b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f350b87479cf47a7836a813889486d2b 2024-12-05T11:58:12,134 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6e262a49461044fca25edb015fdec0a3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6e262a49461044fca25edb015fdec0a3 2024-12-05T11:58:12,135 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/0277ca04c61d449085d37dc6d89d2774 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/0277ca04c61d449085d37dc6d89d2774 2024-12-05T11:58:12,135 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/a7746edf0d394464b7cfdc3bf2c8e957 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/a7746edf0d394464b7cfdc3bf2c8e957 2024-12-05T11:58:12,136 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/57bc404d80a140929db63a1bbb6d5e5d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/57bc404d80a140929db63a1bbb6d5e5d 2024-12-05T11:58:12,137 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d21a0f1259f64e618b8a81d47016623d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/d21a0f1259f64e618b8a81d47016623d 2024-12-05T11:58:12,138 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/bc4ee71f1cc1425fb0a5f80327e2f57b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/bc4ee71f1cc1425fb0a5f80327e2f57b 2024-12-05T11:58:12,139 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/89c55fafc47b4557ae301c8f10bdcedd to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/89c55fafc47b4557ae301c8f10bdcedd 2024-12-05T11:58:12,140 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f352ba5d87ab45f1a0d6251abff1e7ef to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f352ba5d87ab45f1a0d6251abff1e7ef 2024-12-05T11:58:12,140 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/83b0bb7a4441457a89a0d293a18ebe23 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/83b0bb7a4441457a89a0d293a18ebe23 2024-12-05T11:58:12,141 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/1db12eb5a34947358c2364c9850cdf03 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/1db12eb5a34947358c2364c9850cdf03 2024-12-05T11:58:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:12,144 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/73210547eff14fffbc0542ecba290257, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/0bfa9189ed1846afb5c33abcd40c17de, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/a38b14b75c844ece8342f18381f20f12, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/1fcd2dcd451141808119d77d9730a83a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/b37dfc4a5202465d8f25ee5d6e3670ae, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/2f09de2f81ee4b868ed4e5df78a3b543, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/bfe131bbfc2e4d6db452138217f32ef0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/96d9305763f34392b8b77b510b10228c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/201f04cf91214e71975e1f127cc36d20, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/13610819ed3f47358120a3ed84eb26b6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/eea73ecd03804e62b1aa4636dcad6433, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d1680703f6904a72aa8cd866c0953680, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/2e42290099d54627ba68c00f0db854f6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/f81c263c76c249779ba240cfe380f4bd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d8d9511cee5440368291922e9e270623, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/8bcdb0531cb34a218ad6b918a2476342, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/cabc1199328945fc83896b03ddc2f3fc, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/f0100fdfb27e4a5081cd3072330d2ca9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/6cb4856d7fa14aa2aec3bf4695cb1dc3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/66d18cb6fb4d4ff6a2798a52251fd967] to archive 2024-12-05T11:58:12,145 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:58:12,146 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/73210547eff14fffbc0542ecba290257 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/73210547eff14fffbc0542ecba290257 2024-12-05T11:58:12,147 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/0bfa9189ed1846afb5c33abcd40c17de to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/0bfa9189ed1846afb5c33abcd40c17de 2024-12-05T11:58:12,147 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051dd5273e58e7493698cbff4659aea25e_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051dd5273e58e7493698cbff4659aea25e_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:12,147 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/a38b14b75c844ece8342f18381f20f12 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/a38b14b75c844ece8342f18381f20f12 2024-12-05T11:58:12,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/c2b434061ee640f3ad420f223d09833f, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:12,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/c2b434061ee640f3ad420f223d09833f is 175, key is test_row_0/A:col10/1733399889970/Put/seqid=0 2024-12-05T11:58:12,149 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/1fcd2dcd451141808119d77d9730a83a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/1fcd2dcd451141808119d77d9730a83a 2024-12-05T11:58:12,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742231_1407 (size=31255) 2024-12-05T11:58:12,152 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/b37dfc4a5202465d8f25ee5d6e3670ae to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/b37dfc4a5202465d8f25ee5d6e3670ae 2024-12-05T11:58:12,153 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/2f09de2f81ee4b868ed4e5df78a3b543 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/2f09de2f81ee4b868ed4e5df78a3b543 2024-12-05T11:58:12,154 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/bfe131bbfc2e4d6db452138217f32ef0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/bfe131bbfc2e4d6db452138217f32ef0 2024-12-05T11:58:12,155 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/96d9305763f34392b8b77b510b10228c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/96d9305763f34392b8b77b510b10228c 2024-12-05T11:58:12,156 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/201f04cf91214e71975e1f127cc36d20 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/201f04cf91214e71975e1f127cc36d20 2024-12-05T11:58:12,157 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/13610819ed3f47358120a3ed84eb26b6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/13610819ed3f47358120a3ed84eb26b6 2024-12-05T11:58:12,157 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/eea73ecd03804e62b1aa4636dcad6433 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/eea73ecd03804e62b1aa4636dcad6433 2024-12-05T11:58:12,158 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d1680703f6904a72aa8cd866c0953680 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d1680703f6904a72aa8cd866c0953680 2024-12-05T11:58:12,159 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/2e42290099d54627ba68c00f0db854f6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/2e42290099d54627ba68c00f0db854f6 2024-12-05T11:58:12,160 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/f81c263c76c249779ba240cfe380f4bd to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/f81c263c76c249779ba240cfe380f4bd 2024-12-05T11:58:12,161 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d8d9511cee5440368291922e9e270623 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d8d9511cee5440368291922e9e270623 2024-12-05T11:58:12,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399952158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,162 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/8bcdb0531cb34a218ad6b918a2476342 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/8bcdb0531cb34a218ad6b918a2476342 2024-12-05T11:58:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,163 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/cabc1199328945fc83896b03ddc2f3fc to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/cabc1199328945fc83896b03ddc2f3fc 2024-12-05T11:58:12,163 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/f0100fdfb27e4a5081cd3072330d2ca9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/f0100fdfb27e4a5081cd3072330d2ca9 2024-12-05T11:58:12,164 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/6cb4856d7fa14aa2aec3bf4695cb1dc3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/6cb4856d7fa14aa2aec3bf4695cb1dc3 2024-12-05T11:58:12,165 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/66d18cb6fb4d4ff6a2798a52251fd967 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/66d18cb6fb4d4ff6a2798a52251fd967 2024-12-05T11:58:12,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399952159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399952161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399952161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399952162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T11:58:12,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399952267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399952277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399952278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399952278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399952278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399952478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399952487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399952487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399952487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399952487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,552 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/c2b434061ee640f3ad420f223d09833f 2024-12-05T11:58:12,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/c2c2b4585d2942df9373ca20e7d00163 is 50, key is test_row_0/B:col10/1733399889970/Put/seqid=0 2024-12-05T11:58:12,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742232_1408 (size=12301) 2024-12-05T11:58:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T11:58:12,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399952788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399952797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399952797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399952797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399952797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:12,962 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/c2c2b4585d2942df9373ca20e7d00163 2024-12-05T11:58:12,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/8e0c3e691c654b2589191de093937c82 is 50, key is test_row_0/C:col10/1733399889970/Put/seqid=0 2024-12-05T11:58:12,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742233_1409 (size=12301) 2024-12-05T11:58:12,977 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/8e0c3e691c654b2589191de093937c82 2024-12-05T11:58:12,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/c2b434061ee640f3ad420f223d09833f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c2b434061ee640f3ad420f223d09833f 2024-12-05T11:58:12,985 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c2b434061ee640f3ad420f223d09833f, entries=150, sequenceid=312, filesize=30.5 K 2024-12-05T11:58:12,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/c2c2b4585d2942df9373ca20e7d00163 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/c2c2b4585d2942df9373ca20e7d00163 2024-12-05T11:58:12,988 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/c2c2b4585d2942df9373ca20e7d00163, entries=150, sequenceid=312, filesize=12.0 K 2024-12-05T11:58:12,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/8e0c3e691c654b2589191de093937c82 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/8e0c3e691c654b2589191de093937c82 2024-12-05T11:58:12,992 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/8e0c3e691c654b2589191de093937c82, entries=150, sequenceid=312, filesize=12.0 K 2024-12-05T11:58:12,993 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6fb7227437f8ecf2f0f7089b8972c742 in 1270ms, sequenceid=312, compaction requested=false 2024-12-05T11:58:12,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:12,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:12,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-05T11:58:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-05T11:58:12,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-05T11:58:12,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-12-05T11:58:12,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4290 sec 2024-12-05T11:58:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:13,301 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:13,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T11:58:13,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055bb77dcdb5cd4a6b86c3eb99320346aa_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399893300/Put/seqid=0 2024-12-05T11:58:13,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742234_1410 (size=12454) 2024-12-05T11:58:13,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399953315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399953316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399953316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399953322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399953323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399953428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399953428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399953428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399953428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399953437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399953638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399953638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399953638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399953638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399953647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,710 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:13,714 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055bb77dcdb5cd4a6b86c3eb99320346aa_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055bb77dcdb5cd4a6b86c3eb99320346aa_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:13,714 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/6c676e0332974fc6808024fa6c60e2d6, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:13,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/6c676e0332974fc6808024fa6c60e2d6 is 175, key is test_row_0/A:col10/1733399893300/Put/seqid=0 2024-12-05T11:58:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T11:58:13,718 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:13,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-05T11:58:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T11:58:13,720 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:13,721 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:13,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:13,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742235_1411 (size=31255) 2024-12-05T11:58:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T11:58:13,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-05T11:58:13,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:13,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:13,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:13,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:13,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:13,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399953948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399953948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399953948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399953948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:13,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399953957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:14,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-05T11:58:14,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:14,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:14,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:14,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:14,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T11:58:14,127 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/6c676e0332974fc6808024fa6c60e2d6 2024-12-05T11:58:14,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/da076f3cef95451d85c7bbf3331e061d is 50, key is test_row_0/B:col10/1733399893300/Put/seqid=0 2024-12-05T11:58:14,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742236_1412 (size=12301) 2024-12-05T11:58:14,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/da076f3cef95451d85c7bbf3331e061d 2024-12-05T11:58:14,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/b38316f8d8e143b6b23a375496175350 is 50, key is test_row_0/C:col10/1733399893300/Put/seqid=0 2024-12-05T11:58:14,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742237_1413 (size=12301) 2024-12-05T11:58:14,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-05T11:58:14,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:14,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:14,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:14,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:14,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:14,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-05T11:58:14,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:14,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:14,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:14,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:14,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T11:58:14,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46284 deadline: 1733399954457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:14,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46310 deadline: 1733399954457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:14,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1733399954458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:14,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46262 deadline: 1733399954468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:14,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46236 deadline: 1733399954468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:14,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-05T11:58:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:14,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:14,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:14,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/b38316f8d8e143b6b23a375496175350 2024-12-05T11:58:14,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/6c676e0332974fc6808024fa6c60e2d6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/6c676e0332974fc6808024fa6c60e2d6 2024-12-05T11:58:14,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/6c676e0332974fc6808024fa6c60e2d6, entries=150, sequenceid=337, filesize=30.5 K 2024-12-05T11:58:14,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/da076f3cef95451d85c7bbf3331e061d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/da076f3cef95451d85c7bbf3331e061d 2024-12-05T11:58:14,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/da076f3cef95451d85c7bbf3331e061d, entries=150, sequenceid=337, filesize=12.0 K 2024-12-05T11:58:14,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/b38316f8d8e143b6b23a375496175350 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/b38316f8d8e143b6b23a375496175350 2024-12-05T11:58:14,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/b38316f8d8e143b6b23a375496175350, entries=150, sequenceid=337, filesize=12.0 K 2024-12-05T11:58:14,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6fb7227437f8ecf2f0f7089b8972c742 in 1287ms, sequenceid=337, compaction requested=true 2024-12-05T11:58:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:14,588 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fb7227437f8ecf2f0f7089b8972c742:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:14,588 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:14,590 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:14,590 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/A is initiating minor compaction (all files) 2024-12-05T11:58:14,590 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/A in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:14,590 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/1dbe95545354482d85b8b455675db494, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c2b434061ee640f3ad420f223d09833f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/6c676e0332974fc6808024fa6c60e2d6] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=92.2 K 2024-12-05T11:58:14,590 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:14,590 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/1dbe95545354482d85b8b455675db494, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c2b434061ee640f3ad420f223d09833f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/6c676e0332974fc6808024fa6c60e2d6] 2024-12-05T11:58:14,590 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:14,590 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/B is initiating minor compaction (all files) 2024-12-05T11:58:14,591 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dbe95545354482d85b8b455675db494, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733399889309 2024-12-05T11:58:14,591 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/B in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:14,591 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/dabbe08d35c1496abc0aa7ff1688238a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/c2c2b4585d2942df9373ca20e7d00163, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/da076f3cef95451d85c7bbf3331e061d] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=36.7 K 2024-12-05T11:58:14,591 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b434061ee640f3ad420f223d09833f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733399889965 2024-12-05T11:58:14,591 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting dabbe08d35c1496abc0aa7ff1688238a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733399889309 2024-12-05T11:58:14,591 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c676e0332974fc6808024fa6c60e2d6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733399892150 2024-12-05T11:58:14,591 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c2b4585d2942df9373ca20e7d00163, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733399889965 2024-12-05T11:58:14,594 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting da076f3cef95451d85c7bbf3331e061d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733399892150 2024-12-05T11:58:14,607 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:14,613 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#B#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:14,613 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/658ba4c3652e48e78579f0a056c59fc4 is 50, key is test_row_0/B:col10/1733399893300/Put/seqid=0 2024-12-05T11:58:14,622 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205f33b8b5ecedf4b70a8703b187922c857_6fb7227437f8ecf2f0f7089b8972c742 store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:14,623 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205f33b8b5ecedf4b70a8703b187922c857_6fb7227437f8ecf2f0f7089b8972c742, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:14,623 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f33b8b5ecedf4b70a8703b187922c857_6fb7227437f8ecf2f0f7089b8972c742 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:14,636 INFO [Thread-1487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1487. 2024-12-05T11:58:14,636 DEBUG [Thread-1487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:14,636 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:14,636 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:14,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:14,636 INFO [Thread-1481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1481. 2024-12-05T11:58:14,637 DEBUG [Thread-1481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:14,637 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:14,637 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:14,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:14,639 INFO [Thread-1489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1489. 2024-12-05T11:58:14,639 DEBUG [Thread-1489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:14,639 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:14,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-05T11:58:14,639 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:14,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:14,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:14,640 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T11:58:14,641 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-12-05T11:58:14,641 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:14,641 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:14,641 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:14,641 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:14,642 INFO [Thread-1483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1483. 2024-12-05T11:58:14,642 DEBUG [Thread-1483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:14,642 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:14,642 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:14,642 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:14,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742239_1415 (size=4469) 2024-12-05T11:58:14,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742238_1414 (size=12404) 2024-12-05T11:58:14,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205de25af7115b24eebbc8e1aa04133099e_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399893314/Put/seqid=0 2024-12-05T11:58:14,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742240_1416 (size=12454) 2024-12-05T11:58:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T11:58:15,046 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#A#compaction#346 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:15,047 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/15bc3a8f97ab4de3ac4d3f39dc30ae49 is 175, key is test_row_0/A:col10/1733399893300/Put/seqid=0 2024-12-05T11:58:15,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:15,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742241_1417 (size=31358) 2024-12-05T11:58:15,054 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/658ba4c3652e48e78579f0a056c59fc4 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/658ba4c3652e48e78579f0a056c59fc4 2024-12-05T11:58:15,057 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205de25af7115b24eebbc8e1aa04133099e_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205de25af7115b24eebbc8e1aa04133099e_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:15,058 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/15bc3a8f97ab4de3ac4d3f39dc30ae49 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/15bc3a8f97ab4de3ac4d3f39dc30ae49 2024-12-05T11:58:15,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/3648c52442cc4619a620208253cd5308, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:15,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/3648c52442cc4619a620208253cd5308 is 175, key is test_row_0/A:col10/1733399893314/Put/seqid=0 2024-12-05T11:58:15,059 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/B of 6fb7227437f8ecf2f0f7089b8972c742 into 658ba4c3652e48e78579f0a056c59fc4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:15,059 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:15,059 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/B, priority=13, startTime=1733399894588; duration=0sec 2024-12-05T11:58:15,059 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:15,059 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:B 2024-12-05T11:58:15,059 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:15,060 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:15,060 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 6fb7227437f8ecf2f0f7089b8972c742/C is initiating minor compaction (all files) 2024-12-05T11:58:15,060 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fb7227437f8ecf2f0f7089b8972c742/C in TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:15,060 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d6af5e2d6a4747fb9d0f83401f1077d7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/8e0c3e691c654b2589191de093937c82, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/b38316f8d8e143b6b23a375496175350] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp, totalSize=36.7 K 2024-12-05T11:58:15,060 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d6af5e2d6a4747fb9d0f83401f1077d7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733399889309 2024-12-05T11:58:15,060 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e0c3e691c654b2589191de093937c82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733399889965 2024-12-05T11:58:15,061 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting b38316f8d8e143b6b23a375496175350, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733399892150 2024-12-05T11:58:15,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742242_1418 (size=31255) 2024-12-05T11:58:15,062 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/A of 6fb7227437f8ecf2f0f7089b8972c742 into 15bc3a8f97ab4de3ac4d3f39dc30ae49(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:15,062 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:15,062 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/A, priority=13, startTime=1733399894588; duration=0sec 2024-12-05T11:58:15,062 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:15,062 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:A 2024-12-05T11:58:15,066 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fb7227437f8ecf2f0f7089b8972c742#C#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:15,066 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/bf2dbd75e3504334969ac9e5b63d224b is 50, key is test_row_0/C:col10/1733399893300/Put/seqid=0 2024-12-05T11:58:15,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742243_1419 (size=12404) 2024-12-05T11:58:15,464 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/3648c52442cc4619a620208253cd5308 2024-12-05T11:58:15,476 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/bf2dbd75e3504334969ac9e5b63d224b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/bf2dbd75e3504334969ac9e5b63d224b 2024-12-05T11:58:15,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/f3c77136df2249d2844e5a15b2311fac is 50, key is test_row_0/B:col10/1733399893314/Put/seqid=0 2024-12-05T11:58:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. as already flushing 2024-12-05T11:58:15,480 INFO [Thread-1478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1478. 2024-12-05T11:58:15,480 INFO [Thread-1476 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1476. 2024-12-05T11:58:15,480 DEBUG [Thread-1478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:15,480 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:15,480 DEBUG [Thread-1476 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:15,480 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:15,480 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:15,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:15,480 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:15,480 INFO [Thread-1470 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1470. 2024-12-05T11:58:15,480 DEBUG [Thread-1470 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:15,480 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:15,480 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:15,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:15,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:15,481 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fb7227437f8ecf2f0f7089b8972c742/C of 6fb7227437f8ecf2f0f7089b8972c742 into bf2dbd75e3504334969ac9e5b63d224b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:15,481 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:15,481 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., storeName=6fb7227437f8ecf2f0f7089b8972c742/C, priority=13, startTime=1733399894588; duration=0sec 2024-12-05T11:58:15,481 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:15,481 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fb7227437f8ecf2f0f7089b8972c742:C 2024-12-05T11:58:15,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742244_1420 (size=12301) 2024-12-05T11:58:15,482 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/f3c77136df2249d2844e5a15b2311fac 2024-12-05T11:58:15,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/0b5d687820714732a761060e173892a9 is 50, key is test_row_0/C:col10/1733399893314/Put/seqid=0 2024-12-05T11:58:15,489 INFO [Thread-1472 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1472. 2024-12-05T11:58:15,489 DEBUG [Thread-1472 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:15,489 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:15,489 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:15,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:15,489 INFO [Thread-1474 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1474. 2024-12-05T11:58:15,489 DEBUG [Thread-1474 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:15,489 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:15,489 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:15,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:15,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742245_1421 (size=12301) 2024-12-05T11:58:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T11:58:15,892 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/0b5d687820714732a761060e173892a9 2024-12-05T11:58:15,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/3648c52442cc4619a620208253cd5308 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/3648c52442cc4619a620208253cd5308 2024-12-05T11:58:15,909 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/3648c52442cc4619a620208253cd5308, entries=150, sequenceid=349, filesize=30.5 K 2024-12-05T11:58:15,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/f3c77136df2249d2844e5a15b2311fac as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f3c77136df2249d2844e5a15b2311fac 2024-12-05T11:58:15,914 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f3c77136df2249d2844e5a15b2311fac, entries=150, sequenceid=349, filesize=12.0 K 2024-12-05T11:58:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/0b5d687820714732a761060e173892a9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/0b5d687820714732a761060e173892a9 2024-12-05T11:58:15,918 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/0b5d687820714732a761060e173892a9, entries=150, sequenceid=349, filesize=12.0 K 2024-12-05T11:58:15,919 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=33.54 KB/34350 for 6fb7227437f8ecf2f0f7089b8972c742 in 1280ms, sequenceid=349, compaction requested=false 2024-12-05T11:58:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 6fb7227437f8ecf2f0f7089b8972c742: 2024-12-05T11:58:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-05T11:58:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-05T11:58:15,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-05T11:58:15,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1990 sec 2024-12-05T11:58:15,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.2030 sec 2024-12-05T11:58:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T11:58:17,899 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1814 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5440 rows 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1826 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5476 rows 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1794 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5382 rows 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1806 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5416 rows 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1812 2024-12-05T11:58:17,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5434 rows 2024-12-05T11:58:17,901 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:58:17,901 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:58:17,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:17,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:17,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:17,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T11:58:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T11:58:17,940 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399897940"}]},"ts":"1733399897940"} 2024-12-05T11:58:17,942 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T11:58:17,942 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T11:58:17,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:58:17,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, UNASSIGN}] 2024-12-05T11:58:17,947 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, UNASSIGN 2024-12-05T11:58:17,949 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=6fb7227437f8ecf2f0f7089b8972c742, regionState=CLOSING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:58:17,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, UNASSIGN because future has completed 2024-12-05T11:58:17,951 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:58:17,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:58:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T11:58:18,105 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:18,105 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:58:18,105 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 6fb7227437f8ecf2f0f7089b8972c742, disabling compactions & flushes 2024-12-05T11:58:18,105 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:18,105 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:18,105 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. after waiting 0 ms 2024-12-05T11:58:18,105 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:18,106 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 6fb7227437f8ecf2f0f7089b8972c742 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-05T11:58:18,115 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205476bcd15746249189f664b37ecf2d49c_6fb7227437f8ecf2f0f7089b8972c742 is 50, key is test_row_0/A:col10/1733399895488/Put/seqid=0 2024-12-05T11:58:18,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742246_1422 (size=12454) 2024-12-05T11:58:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T11:58:18,522 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:18,533 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205476bcd15746249189f664b37ecf2d49c_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205476bcd15746249189f664b37ecf2d49c_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:18,534 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/12b87d2d5777429881d2b154b853d928, store: [table=TestAcidGuarantees family=A region=6fb7227437f8ecf2f0f7089b8972c742] 2024-12-05T11:58:18,535 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/12b87d2d5777429881d2b154b853d928 is 175, key is test_row_0/A:col10/1733399895488/Put/seqid=0 2024-12-05T11:58:18,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742247_1423 (size=31255) 2024-12-05T11:58:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T11:58:18,940 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/12b87d2d5777429881d2b154b853d928 2024-12-05T11:58:18,959 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/6bf23f1231c24ec9808f07c202485029 is 50, key is test_row_0/B:col10/1733399895488/Put/seqid=0 2024-12-05T11:58:18,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742248_1424 (size=12301) 2024-12-05T11:58:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T11:58:19,363 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/6bf23f1231c24ec9808f07c202485029 2024-12-05T11:58:19,373 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/055ba36d9e1a44ac81936fd52e719413 is 50, key is test_row_0/C:col10/1733399895488/Put/seqid=0 2024-12-05T11:58:19,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742249_1425 (size=12301) 2024-12-05T11:58:19,777 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/055ba36d9e1a44ac81936fd52e719413 2024-12-05T11:58:19,787 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/A/12b87d2d5777429881d2b154b853d928 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/12b87d2d5777429881d2b154b853d928 2024-12-05T11:58:19,790 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/12b87d2d5777429881d2b154b853d928, entries=150, sequenceid=360, filesize=30.5 K 2024-12-05T11:58:19,791 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/B/6bf23f1231c24ec9808f07c202485029 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6bf23f1231c24ec9808f07c202485029 2024-12-05T11:58:19,794 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6bf23f1231c24ec9808f07c202485029, entries=150, sequenceid=360, filesize=12.0 K 2024-12-05T11:58:19,795 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/.tmp/C/055ba36d9e1a44ac81936fd52e719413 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/055ba36d9e1a44ac81936fd52e719413 2024-12-05T11:58:19,798 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/055ba36d9e1a44ac81936fd52e719413, entries=150, sequenceid=360, filesize=12.0 K 2024-12-05T11:58:19,799 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 6fb7227437f8ecf2f0f7089b8972c742 in 1693ms, sequenceid=360, compaction requested=true 2024-12-05T11:58:19,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/1dbe95545354482d85b8b455675db494, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c2b434061ee640f3ad420f223d09833f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/6c676e0332974fc6808024fa6c60e2d6] to archive 2024-12-05T11:58:19,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:58:19,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/1dbe95545354482d85b8b455675db494 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/1dbe95545354482d85b8b455675db494 2024-12-05T11:58:19,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c2b434061ee640f3ad420f223d09833f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/c2b434061ee640f3ad420f223d09833f 2024-12-05T11:58:19,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/6c676e0332974fc6808024fa6c60e2d6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/6c676e0332974fc6808024fa6c60e2d6 2024-12-05T11:58:19,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/dabbe08d35c1496abc0aa7ff1688238a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/c2c2b4585d2942df9373ca20e7d00163, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/da076f3cef95451d85c7bbf3331e061d] to archive 2024-12-05T11:58:19,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:58:19,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/dabbe08d35c1496abc0aa7ff1688238a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/dabbe08d35c1496abc0aa7ff1688238a 2024-12-05T11:58:19,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/c2c2b4585d2942df9373ca20e7d00163 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/c2c2b4585d2942df9373ca20e7d00163 2024-12-05T11:58:19,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/da076f3cef95451d85c7bbf3331e061d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/da076f3cef95451d85c7bbf3331e061d 2024-12-05T11:58:19,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d6af5e2d6a4747fb9d0f83401f1077d7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/8e0c3e691c654b2589191de093937c82, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/b38316f8d8e143b6b23a375496175350] to archive 2024-12-05T11:58:19,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:58:19,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d6af5e2d6a4747fb9d0f83401f1077d7 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/d6af5e2d6a4747fb9d0f83401f1077d7 2024-12-05T11:58:19,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/8e0c3e691c654b2589191de093937c82 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/8e0c3e691c654b2589191de093937c82 2024-12-05T11:58:19,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/b38316f8d8e143b6b23a375496175350 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/b38316f8d8e143b6b23a375496175350 2024-12-05T11:58:19,820 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/recovered.edits/363.seqid, newMaxSeqId=363, maxSeqId=4 2024-12-05T11:58:19,821 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. 2024-12-05T11:58:19,821 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 6fb7227437f8ecf2f0f7089b8972c742: Waiting for close lock at 1733399898105Running coprocessor pre-close hooks at 1733399898105Disabling compacts and flushes for region at 1733399898105Disabling writes for close at 1733399898105Obtaining lock to block concurrent updates at 1733399898106 (+1 ms)Preparing flush snapshotting stores in 6fb7227437f8ecf2f0f7089b8972c742 at 1733399898106Finished memstore snapshotting TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733399898106Flushing stores of TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742. at 1733399898107 (+1 ms)Flushing 6fb7227437f8ecf2f0f7089b8972c742/A: creating writer at 1733399898107Flushing mob file 6fb7227437f8ecf2f0f7089b8972c742/A: appending metadata at 1733399898115 (+8 ms)Flushing mob file 6fb7227437f8ecf2f0f7089b8972c742/A: closing flushed file at 1733399898115Flushing 6fb7227437f8ecf2f0f7089b8972c742/A: appending metadata at 1733399898534 (+419 ms)Flushing 6fb7227437f8ecf2f0f7089b8972c742/A: closing flushed file at 1733399898534Flushing 6fb7227437f8ecf2f0f7089b8972c742/B: creating writer at 1733399898956 (+422 ms)Flushing 6fb7227437f8ecf2f0f7089b8972c742/B: appending metadata at 1733399898958 (+2 ms)Flushing 6fb7227437f8ecf2f0f7089b8972c742/B: closing flushed file at 1733399898959 (+1 ms)Flushing 6fb7227437f8ecf2f0f7089b8972c742/C: creating writer at 1733399899370 (+411 ms)Flushing 6fb7227437f8ecf2f0f7089b8972c742/C: appending metadata at 1733399899372 (+2 ms)Flushing 6fb7227437f8ecf2f0f7089b8972c742/C: closing flushed file at 1733399899372Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47a5492a: reopening flushed file at 1733399899786 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@763ae2ff: reopening flushed file at 1733399899791 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e60d803: reopening flushed file at 1733399899794 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 6fb7227437f8ecf2f0f7089b8972c742 in 1693ms, sequenceid=360, compaction requested=true at 1733399899799 (+5 ms)Writing region close event to WAL at 1733399899817 (+18 ms)Running coprocessor post-close hooks at 1733399899821 (+4 ms)Closed at 1733399899821 2024-12-05T11:58:19,823 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:19,824 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=6fb7227437f8ecf2f0f7089b8972c742, regionState=CLOSED 2024-12-05T11:58:19,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:58:19,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-05T11:58:19,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 6fb7227437f8ecf2f0f7089b8972c742, server=80666a11a09f,40121,1733399769572 in 1.8760 sec 2024-12-05T11:58:19,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-05T11:58:19,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fb7227437f8ecf2f0f7089b8972c742, UNASSIGN in 1.8830 sec 2024-12-05T11:58:19,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-05T11:58:19,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8900 sec 2024-12-05T11:58:19,838 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399899838"}]},"ts":"1733399899838"} 2024-12-05T11:58:19,840 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T11:58:19,841 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T11:58:19,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9390 sec 2024-12-05T11:58:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T11:58:20,089 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:20,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T11:58:20,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:20,095 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:20,097 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T11:58:20,101 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,103 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/recovered.edits] 2024-12-05T11:58:20,106 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/12b87d2d5777429881d2b154b853d928 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/12b87d2d5777429881d2b154b853d928 2024-12-05T11:58:20,107 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/15bc3a8f97ab4de3ac4d3f39dc30ae49 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/15bc3a8f97ab4de3ac4d3f39dc30ae49 2024-12-05T11:58:20,108 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/3648c52442cc4619a620208253cd5308 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/A/3648c52442cc4619a620208253cd5308 2024-12-05T11:58:20,111 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/658ba4c3652e48e78579f0a056c59fc4 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/658ba4c3652e48e78579f0a056c59fc4 2024-12-05T11:58:20,112 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6bf23f1231c24ec9808f07c202485029 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/6bf23f1231c24ec9808f07c202485029 2024-12-05T11:58:20,114 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f3c77136df2249d2844e5a15b2311fac to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/B/f3c77136df2249d2844e5a15b2311fac 2024-12-05T11:58:20,117 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/055ba36d9e1a44ac81936fd52e719413 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/055ba36d9e1a44ac81936fd52e719413 2024-12-05T11:58:20,119 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/0b5d687820714732a761060e173892a9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/0b5d687820714732a761060e173892a9 2024-12-05T11:58:20,121 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/bf2dbd75e3504334969ac9e5b63d224b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/C/bf2dbd75e3504334969ac9e5b63d224b 2024-12-05T11:58:20,125 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/recovered.edits/363.seqid to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742/recovered.edits/363.seqid 2024-12-05T11:58:20,125 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,126 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T11:58:20,126 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T11:58:20,127 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T11:58:20,130 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120503717168a55c498c953f561a3e406160_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120503717168a55c498c953f561a3e406160_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,131 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050e5db4aa5ce142fab283ab070babfe31_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050e5db4aa5ce142fab283ab070babfe31_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,132 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051c7ad7a3052847e294d71c712b47b2b8_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051c7ad7a3052847e294d71c712b47b2b8_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,133 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051dd5273e58e7493698cbff4659aea25e_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051dd5273e58e7493698cbff4659aea25e_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,134 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051de2279332b4419abcb87c6a54a8e8aa_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051de2279332b4419abcb87c6a54a8e8aa_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,135 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120540c8cc2e4bc24342a8e9b17f0440ef6b_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120540c8cc2e4bc24342a8e9b17f0440ef6b_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,136 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120541b68d841feb407aa8d3f3e3fbd4d56a_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120541b68d841feb407aa8d3f3e3fbd4d56a_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,137 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054570bea3511242d691b4e64431826c52_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054570bea3511242d691b4e64431826c52_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,138 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205476bcd15746249189f664b37ecf2d49c_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205476bcd15746249189f664b37ecf2d49c_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,140 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055bb77dcdb5cd4a6b86c3eb99320346aa_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055bb77dcdb5cd4a6b86c3eb99320346aa_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,141 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120570f6a162f56145b4a80b85be4b281d65_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120570f6a162f56145b4a80b85be4b281d65_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,142 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205744ea6d5c91f4da5a0538d6ff815c5ae_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205744ea6d5c91f4da5a0538d6ff815c5ae_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,144 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057974883a774e4d448648c14d458309e9_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057974883a774e4d448648c14d458309e9_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,145 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057c740b3e2b7e44f9b4ece09a82ccc09e_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057c740b3e2b7e44f9b4ece09a82ccc09e_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,146 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a9e893c1bba84fb2ac319a591517ca83_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a9e893c1bba84fb2ac319a591517ca83_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,147 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ce38e928e0694bbf8e6c096f342a0738_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ce38e928e0694bbf8e6c096f342a0738_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,149 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205de25af7115b24eebbc8e1aa04133099e_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205de25af7115b24eebbc8e1aa04133099e_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,150 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e2f3cee715ce4c4d853dc829960ab91f_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e2f3cee715ce4c4d853dc829960ab91f_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,151 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ec69c4d099594914a2057381fe0726ca_6fb7227437f8ecf2f0f7089b8972c742 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ec69c4d099594914a2057381fe0726ca_6fb7227437f8ecf2f0f7089b8972c742 2024-12-05T11:58:20,152 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T11:58:20,154 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:20,156 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T11:58:20,157 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T11:58:20,158 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:20,158 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T11:58:20,158 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733399900158"}]},"ts":"9223372036854775807"} 2024-12-05T11:58:20,160 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T11:58:20,160 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6fb7227437f8ecf2f0f7089b8972c742, NAME => 'TestAcidGuarantees,,1733399871499.6fb7227437f8ecf2f0f7089b8972c742.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T11:58:20,160 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T11:58:20,160 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733399900160"}]},"ts":"9223372036854775807"} 2024-12-05T11:58:20,161 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T11:58:20,162 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:20,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 72 msec 2024-12-05T11:58:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T11:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T11:58:20,208 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:20,217 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 237) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1849376751_22 at /127.0.0.1:37744 [Waiting for operation #304] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1451530749_22 at /127.0.0.1:37748 [Waiting for operation #317] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=344 (was 276) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7271 (was 7749) 2024-12-05T11:58:20,224 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=344, ProcessCount=11, AvailableMemoryMB=7270 2024-12-05T11:58:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:58:20,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:58:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:20,227 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T11:58:20,227 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:20,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-12-05T11:58:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-05T11:58:20,228 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T11:58:20,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742250_1426 (size=959) 2024-12-05T11:58:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-05T11:58:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-05T11:58:20,638 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c 2024-12-05T11:58:20,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742251_1427 (size=53) 2024-12-05T11:58:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-05T11:58:21,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:58:21,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e6ea33c1931a2d534a8c12ecd9fa9d12, disabling compactions & flushes 2024-12-05T11:58:21,049 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:21,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:21,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. after waiting 0 ms 2024-12-05T11:58:21,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:21,049 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:21,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e6ea33c1931a2d534a8c12ecd9fa9d12: Waiting for close lock at 1733399901049Disabling compacts and flushes for region at 1733399901049Disabling writes for close at 1733399901049Writing region close event to WAL at 1733399901049Closed at 1733399901049 2024-12-05T11:58:21,050 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T11:58:21,050 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733399901050"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733399901050"}]},"ts":"1733399901050"} 2024-12-05T11:58:21,052 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T11:58:21,052 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T11:58:21,053 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399901052"}]},"ts":"1733399901052"} 2024-12-05T11:58:21,054 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T11:58:21,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6ea33c1931a2d534a8c12ecd9fa9d12, ASSIGN}] 2024-12-05T11:58:21,055 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6ea33c1931a2d534a8c12ecd9fa9d12, ASSIGN 2024-12-05T11:58:21,056 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6ea33c1931a2d534a8c12ecd9fa9d12, ASSIGN; state=OFFLINE, location=80666a11a09f,40121,1733399769572; forceNewPlan=false, retain=false 2024-12-05T11:58:21,207 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=e6ea33c1931a2d534a8c12ecd9fa9d12, regionState=OPENING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:58:21,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6ea33c1931a2d534a8c12ecd9fa9d12, ASSIGN because future has completed 2024-12-05T11:58:21,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:58:21,363 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:21,363 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:58:21,364 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,364 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:58:21,364 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,364 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,365 INFO [StoreOpener-e6ea33c1931a2d534a8c12ecd9fa9d12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,366 INFO [StoreOpener-e6ea33c1931a2d534a8c12ecd9fa9d12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6ea33c1931a2d534a8c12ecd9fa9d12 columnFamilyName A 2024-12-05T11:58:21,367 DEBUG [StoreOpener-e6ea33c1931a2d534a8c12ecd9fa9d12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:21,367 INFO [StoreOpener-e6ea33c1931a2d534a8c12ecd9fa9d12-1 {}] regionserver.HStore(327): Store=e6ea33c1931a2d534a8c12ecd9fa9d12/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:58:21,367 INFO [StoreOpener-e6ea33c1931a2d534a8c12ecd9fa9d12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-05T11:58:21,368 INFO [StoreOpener-e6ea33c1931a2d534a8c12ecd9fa9d12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6ea33c1931a2d534a8c12ecd9fa9d12 columnFamilyName B 2024-12-05T11:58:21,368 DEBUG [StoreOpener-e6ea33c1931a2d534a8c12ecd9fa9d12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:21,369 INFO [StoreOpener-e6ea33c1931a2d534a8c12ecd9fa9d12-1 {}] regionserver.HStore(327): Store=e6ea33c1931a2d534a8c12ecd9fa9d12/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:58:21,369 INFO [StoreOpener-e6ea33c1931a2d534a8c12ecd9fa9d12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,370 INFO [StoreOpener-e6ea33c1931a2d534a8c12ecd9fa9d12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6ea33c1931a2d534a8c12ecd9fa9d12 columnFamilyName C 2024-12-05T11:58:21,370 DEBUG [StoreOpener-e6ea33c1931a2d534a8c12ecd9fa9d12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:21,370 INFO [StoreOpener-e6ea33c1931a2d534a8c12ecd9fa9d12-1 {}] regionserver.HStore(327): Store=e6ea33c1931a2d534a8c12ecd9fa9d12/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:58:21,370 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,370 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,371 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,371 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,371 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,372 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:58:21,372 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,374 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:58:21,374 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened e6ea33c1931a2d534a8c12ecd9fa9d12; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74900376, jitterRate=0.11610257625579834}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:58:21,374 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:21,375 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for e6ea33c1931a2d534a8c12ecd9fa9d12: Running coprocessor pre-open hook at 1733399901364Writing region info on filesystem at 1733399901364Initializing all the Stores at 1733399901365 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399901365Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399901365Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399901365Cleaning up temporary data from old regions at 1733399901371 (+6 ms)Running coprocessor post-open hooks at 1733399901374 (+3 ms)Region opened successfully at 1733399901374 2024-12-05T11:58:21,375 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., pid=118, masterSystemTime=1733399901360 2024-12-05T11:58:21,376 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:21,376 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:21,377 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=e6ea33c1931a2d534a8c12ecd9fa9d12, regionState=OPEN, openSeqNum=2, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:58:21,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:58:21,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-05T11:58:21,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 in 169 msec 2024-12-05T11:58:21,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-12-05T11:58:21,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6ea33c1931a2d534a8c12ecd9fa9d12, ASSIGN in 326 msec 2024-12-05T11:58:21,382 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T11:58:21,382 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399901382"}]},"ts":"1733399901382"} 2024-12-05T11:58:21,383 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T11:58:21,384 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T11:58:21,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1580 sec 2024-12-05T11:58:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-05T11:58:22,378 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:22,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ad78a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:22,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:22,380 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@579e1388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:22,381 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2587c82d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,382 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:22,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:22,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:22,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:22,385 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:58:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 80666a11a09f,33741,1733399768894 2024-12-05T11:58:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7ade3d77 2024-12-05T11:58:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:58:22,387 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:58:22,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:22,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:22,391 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@440d0867, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:22,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:22,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:22,392 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:22,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67e82316, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:22,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:22,395 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ee3d2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:22,396 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7f7b5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,398 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:22,398 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:22,399 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fed3ab5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:22,400 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:22,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b0bf760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:22,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:22,403 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5bed80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:22,405 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a319c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:22,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:22,407 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff2dc52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:22,409 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa2c679, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:22,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:22,411 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b8d7e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:22,413 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194affa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:22,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:22,415 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587a6a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:22,417 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1740a2c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:22,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:22,419 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56693319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:22,420 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c502f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:22,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:22,422 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@627bfa54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:22,424 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2aa16e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:22,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:22,426 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595c49e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:22,427 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f68d45d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:22,428 DEBUG [Thread-1941 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:22,428 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:22,429 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:22,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:22,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:22,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:22,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:22,431 DEBUG [Thread-1936 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:22,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:58:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:58:22,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:58:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:58:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:22,440 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:22,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T11:58:22,446 DEBUG [Thread-1947 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:22,447 DEBUG [Thread-1943 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:22,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:22,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:58:22,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:58:22,455 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:22,455 DEBUG [Thread-1928 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:22,456 DEBUG [Thread-1928 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:22,456 DEBUG [Thread-1928 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:22,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:22,459 DEBUG [Thread-1932 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:22,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:22,459 DEBUG [Thread-1939 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:58:22,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:22,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:22,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:22,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:22,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:22,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-05T11:58:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:58:22,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:58:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T11:58:22,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2] 2024-12-05T11:58:22,475 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:22,482 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:22,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:22,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/8f08950b7d9e4e34842a9b84f82819c2 is 50, key is test_row_0/A:col10/1733399902440/Put/seqid=0 2024-12-05T11:58:22,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399962494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399962496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742252_1428 (size=12001) 2024-12-05T11:58:22,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/8f08950b7d9e4e34842a9b84f82819c2 2024-12-05T11:58:22,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399962499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49020 deadline: 1733399962504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399962503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/4fa2c613d50b48a49695e9835996262a is 50, key is test_row_0/B:col10/1733399902440/Put/seqid=0 2024-12-05T11:58:22,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742253_1429 (size=12001) 2024-12-05T11:58:22,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/4fa2c613d50b48a49695e9835996262a 2024-12-05T11:58:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T11:58:22,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/c841a4102438483a8de9ece2bac24ac0 is 50, key is test_row_0/C:col10/1733399902440/Put/seqid=0 2024-12-05T11:58:22,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742254_1430 (size=12001) 2024-12-05T11:58:22,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/c841a4102438483a8de9ece2bac24ac0 2024-12-05T11:58:22,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/8f08950b7d9e4e34842a9b84f82819c2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/8f08950b7d9e4e34842a9b84f82819c2 2024-12-05T11:58:22,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/8f08950b7d9e4e34842a9b84f82819c2, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T11:58:22,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/4fa2c613d50b48a49695e9835996262a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4fa2c613d50b48a49695e9835996262a 2024-12-05T11:58:22,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4fa2c613d50b48a49695e9835996262a, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T11:58:22,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/c841a4102438483a8de9ece2bac24ac0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c841a4102438483a8de9ece2bac24ac0 2024-12-05T11:58:22,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399962618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399962618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399962618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c841a4102438483a8de9ece2bac24ac0, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T11:58:22,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 187ms, sequenceid=14, compaction requested=false 2024-12-05T11:58:22,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:22,632 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:22,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T11:58:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:22,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T11:58:22,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:22,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:22,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:22,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:22,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:22,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/dcf7416e32b04286b3563b5590ff6620 is 50, key is test_row_0/A:col10/1733399902496/Put/seqid=0 2024-12-05T11:58:22,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742255_1431 (size=14341) 2024-12-05T11:58:22,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/dcf7416e32b04286b3563b5590ff6620 2024-12-05T11:58:22,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/147defc5dde6416c8624ba80c2313f5b is 50, key is test_row_0/B:col10/1733399902496/Put/seqid=0 2024-12-05T11:58:22,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49020 deadline: 1733399962671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399962676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742256_1432 (size=12001) 2024-12-05T11:58:22,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/147defc5dde6416c8624ba80c2313f5b 2024-12-05T11:58:22,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/f84db19094ce4d3e84f0ccc256e9a916 is 50, key is test_row_0/C:col10/1733399902496/Put/seqid=0 2024-12-05T11:58:22,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742257_1433 (size=12001) 2024-12-05T11:58:22,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/f84db19094ce4d3e84f0ccc256e9a916 2024-12-05T11:58:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/dcf7416e32b04286b3563b5590ff6620 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/dcf7416e32b04286b3563b5590ff6620 2024-12-05T11:58:22,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/dcf7416e32b04286b3563b5590ff6620, entries=200, sequenceid=38, filesize=14.0 K 2024-12-05T11:58:22,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/147defc5dde6416c8624ba80c2313f5b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/147defc5dde6416c8624ba80c2313f5b 2024-12-05T11:58:22,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/147defc5dde6416c8624ba80c2313f5b, entries=150, sequenceid=38, filesize=11.7 K 2024-12-05T11:58:22,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/f84db19094ce4d3e84f0ccc256e9a916 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/f84db19094ce4d3e84f0ccc256e9a916 2024-12-05T11:58:22,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/f84db19094ce4d3e84f0ccc256e9a916, entries=150, sequenceid=38, filesize=11.7 K 2024-12-05T11:58:22,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 123ms, sequenceid=38, compaction requested=false 2024-12-05T11:58:22,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T11:58:22,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T11:58:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:22,789 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T11:58:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:22,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:22,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/9a0abc45583e4583814e01b9b44b418b is 50, key is test_row_0/A:col10/1733399902653/Put/seqid=0 2024-12-05T11:58:22,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742258_1434 (size=12001) 2024-12-05T11:58:22,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399962846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399962848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399962849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399962850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49020 deadline: 1733399962851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399962968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399962968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399962968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399962968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:22,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49020 deadline: 1733399962968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T11:58:23,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399963178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:23,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399963178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:23,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399963178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399963178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:23,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49020 deadline: 1733399963178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:23,220 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/9a0abc45583e4583814e01b9b44b418b 2024-12-05T11:58:23,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/5dbcb75ca9af49f087e90dd0f02103ac is 50, key is test_row_0/B:col10/1733399902653/Put/seqid=0 2024-12-05T11:58:23,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742259_1435 (size=12001) 2024-12-05T11:58:23,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399963488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399963488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:23,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399963488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:23,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:23,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49020 deadline: 1733399963497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399963497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T11:58:23,631 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/5dbcb75ca9af49f087e90dd0f02103ac 2024-12-05T11:58:23,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/2858f591a08f4fc7968fb1916b466334 is 50, key is test_row_0/C:col10/1733399902653/Put/seqid=0 2024-12-05T11:58:23,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742260_1436 (size=12001) 2024-12-05T11:58:23,652 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/2858f591a08f4fc7968fb1916b466334 2024-12-05T11:58:23,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/9a0abc45583e4583814e01b9b44b418b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/9a0abc45583e4583814e01b9b44b418b 2024-12-05T11:58:23,661 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/9a0abc45583e4583814e01b9b44b418b, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T11:58:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/5dbcb75ca9af49f087e90dd0f02103ac as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/5dbcb75ca9af49f087e90dd0f02103ac 2024-12-05T11:58:23,668 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/5dbcb75ca9af49f087e90dd0f02103ac, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T11:58:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/2858f591a08f4fc7968fb1916b466334 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/2858f591a08f4fc7968fb1916b466334 2024-12-05T11:58:23,677 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/2858f591a08f4fc7968fb1916b466334, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T11:58:23,678 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 889ms, sequenceid=50, compaction requested=true 2024-12-05T11:58:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-05T11:58:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-05T11:58:23,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-05T11:58:23,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1980 sec 2024-12-05T11:58:23,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.2260 sec 2024-12-05T11:58:24,004 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:24,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T11:58:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:24,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/6766d2bfa7384faca56520228f6b24cf is 50, key is test_row_0/A:col10/1733399904003/Put/seqid=0 2024-12-05T11:58:24,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742261_1437 (size=12001) 2024-12-05T11:58:24,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49020 deadline: 1733399964018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399964018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399964026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399964027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399964028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399964138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399964138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399964138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399964348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399964348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399964348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/6766d2bfa7384faca56520228f6b24cf 2024-12-05T11:58:24,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/b21203dd86514ac0b81c34b9e4580944 is 50, key is test_row_0/B:col10/1733399904003/Put/seqid=0 2024-12-05T11:58:24,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742262_1438 (size=12001) 2024-12-05T11:58:24,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/b21203dd86514ac0b81c34b9e4580944 2024-12-05T11:58:24,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/0e5ef276cdf744d6b37afb295255a68a is 50, key is test_row_0/C:col10/1733399904003/Put/seqid=0 2024-12-05T11:58:24,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742263_1439 (size=12001) 2024-12-05T11:58:24,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/0e5ef276cdf744d6b37afb295255a68a 2024-12-05T11:58:24,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/6766d2bfa7384faca56520228f6b24cf as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/6766d2bfa7384faca56520228f6b24cf 2024-12-05T11:58:24,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/6766d2bfa7384faca56520228f6b24cf, entries=150, sequenceid=76, filesize=11.7 K 2024-12-05T11:58:24,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/b21203dd86514ac0b81c34b9e4580944 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/b21203dd86514ac0b81c34b9e4580944 2024-12-05T11:58:24,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/b21203dd86514ac0b81c34b9e4580944, entries=150, sequenceid=76, filesize=11.7 K 2024-12-05T11:58:24,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/0e5ef276cdf744d6b37afb295255a68a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/0e5ef276cdf744d6b37afb295255a68a 2024-12-05T11:58:24,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/0e5ef276cdf744d6b37afb295255a68a, entries=150, sequenceid=76, filesize=11.7 K 2024-12-05T11:58:24,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 474ms, sequenceid=76, compaction requested=true 2024-12-05T11:58:24,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:24,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:24,478 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:24,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:24,478 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:24,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:24,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:24,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:24,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:24,480 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:24,480 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/A is initiating minor compaction (all files) 2024-12-05T11:58:24,480 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/A in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:24,480 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/8f08950b7d9e4e34842a9b84f82819c2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/dcf7416e32b04286b3563b5590ff6620, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/9a0abc45583e4583814e01b9b44b418b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/6766d2bfa7384faca56520228f6b24cf] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=49.2 K 2024-12-05T11:58:24,481 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:24,481 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/B is initiating minor compaction (all files) 2024-12-05T11:58:24,481 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/B in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:24,481 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4fa2c613d50b48a49695e9835996262a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/147defc5dde6416c8624ba80c2313f5b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/5dbcb75ca9af49f087e90dd0f02103ac, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/b21203dd86514ac0b81c34b9e4580944] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=46.9 K 2024-12-05T11:58:24,481 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f08950b7d9e4e34842a9b84f82819c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733399902438 2024-12-05T11:58:24,481 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fa2c613d50b48a49695e9835996262a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733399902438 2024-12-05T11:58:24,481 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcf7416e32b04286b3563b5590ff6620, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733399902483 2024-12-05T11:58:24,482 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 147defc5dde6416c8624ba80c2313f5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733399902483 2024-12-05T11:58:24,482 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a0abc45583e4583814e01b9b44b418b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733399902653 2024-12-05T11:58:24,482 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dbcb75ca9af49f087e90dd0f02103ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733399902653 2024-12-05T11:58:24,482 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6766d2bfa7384faca56520228f6b24cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733399902843 2024-12-05T11:58:24,482 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting b21203dd86514ac0b81c34b9e4580944, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733399902843 2024-12-05T11:58:24,493 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#B#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:24,494 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/02d7dc721607467c9367c9cb50ad4fff is 50, key is test_row_0/B:col10/1733399904003/Put/seqid=0 2024-12-05T11:58:24,497 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#A#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:24,497 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/3a6711ddfc8543b38e2ff231df674184 is 50, key is test_row_0/A:col10/1733399904003/Put/seqid=0 2024-12-05T11:58:24,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742264_1440 (size=12139) 2024-12-05T11:58:24,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742265_1441 (size=12139) 2024-12-05T11:58:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T11:58:24,618 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:24,618 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-05T11:58:24,620 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T11:58:24,620 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:24,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:24,662 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:24,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T11:58:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:24,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/a914d4d6ea9a435193aaeaccf691e0d5 is 50, key is test_row_0/A:col10/1733399904005/Put/seqid=0 2024-12-05T11:58:24,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742266_1442 (size=14341) 2024-12-05T11:58:24,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/a914d4d6ea9a435193aaeaccf691e0d5 2024-12-05T11:58:24,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/4e5826ea20f34bbbaa09e6ab7031988f is 50, key is test_row_0/B:col10/1733399904005/Put/seqid=0 2024-12-05T11:58:24,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742267_1443 (size=12001) 2024-12-05T11:58:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T11:58:24,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399964746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399964750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399964758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T11:58:24,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:24,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:24,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:24,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:24,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399964869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399964869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399964869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:24,926 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/3a6711ddfc8543b38e2ff231df674184 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/3a6711ddfc8543b38e2ff231df674184 2024-12-05T11:58:24,927 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/02d7dc721607467c9367c9cb50ad4fff as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/02d7dc721607467c9367c9cb50ad4fff 2024-12-05T11:58:24,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T11:58:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:24,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:24,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T11:58:24,940 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/B of e6ea33c1931a2d534a8c12ecd9fa9d12 into 02d7dc721607467c9367c9cb50ad4fff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:24,940 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:24,940 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/B, priority=12, startTime=1733399904478; duration=0sec 2024-12-05T11:58:24,940 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:24,940 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:B 2024-12-05T11:58:24,940 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:24,941 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/A of e6ea33c1931a2d534a8c12ecd9fa9d12 into 3a6711ddfc8543b38e2ff231df674184(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:24,941 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:24,941 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/A, priority=12, startTime=1733399904478; duration=0sec 2024-12-05T11:58:24,941 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:24,941 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:A 2024-12-05T11:58:24,942 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:24,942 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/C is initiating minor compaction (all files) 2024-12-05T11:58:24,942 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/C in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:24,942 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c841a4102438483a8de9ece2bac24ac0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/f84db19094ce4d3e84f0ccc256e9a916, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/2858f591a08f4fc7968fb1916b466334, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/0e5ef276cdf744d6b37afb295255a68a] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=46.9 K 2024-12-05T11:58:24,942 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c841a4102438483a8de9ece2bac24ac0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733399902438 2024-12-05T11:58:24,943 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting f84db19094ce4d3e84f0ccc256e9a916, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733399902483 2024-12-05T11:58:24,943 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2858f591a08f4fc7968fb1916b466334, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733399902653 2024-12-05T11:58:24,943 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e5ef276cdf744d6b37afb295255a68a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733399902843 2024-12-05T11:58:24,978 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:24,979 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/4d04e0866a6548fd935c00c3e6534e75 is 50, key is test_row_0/C:col10/1733399904003/Put/seqid=0 2024-12-05T11:58:24,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742268_1444 (size=12139) 2024-12-05T11:58:25,000 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/4d04e0866a6548fd935c00c3e6534e75 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/4d04e0866a6548fd935c00c3e6534e75 2024-12-05T11:58:25,007 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/C of e6ea33c1931a2d534a8c12ecd9fa9d12 into 4d04e0866a6548fd935c00c3e6534e75(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:25,007 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:25,007 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/C, priority=12, startTime=1733399904478; duration=0sec 2024-12-05T11:58:25,008 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:25,008 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:C 2024-12-05T11:58:25,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399965038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49020 deadline: 1733399965038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,039 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in e6ea33c1931a2d534a8c12ecd9fa9d12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,041 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in e6ea33c1931a2d534a8c12ecd9fa9d12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T11:58:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:25,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:25,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399965088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399965088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/4e5826ea20f34bbbaa09e6ab7031988f 2024-12-05T11:58:25,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399965089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/299a76c94bbd40b3afd80c11fd42acb7 is 50, key is test_row_0/C:col10/1733399904005/Put/seqid=0 2024-12-05T11:58:25,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742269_1445 (size=12001) 2024-12-05T11:58:25,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/299a76c94bbd40b3afd80c11fd42acb7 2024-12-05T11:58:25,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/a914d4d6ea9a435193aaeaccf691e0d5 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/a914d4d6ea9a435193aaeaccf691e0d5 2024-12-05T11:58:25,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/a914d4d6ea9a435193aaeaccf691e0d5, entries=200, sequenceid=89, filesize=14.0 K 2024-12-05T11:58:25,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/4e5826ea20f34bbbaa09e6ab7031988f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4e5826ea20f34bbbaa09e6ab7031988f 2024-12-05T11:58:25,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4e5826ea20f34bbbaa09e6ab7031988f, entries=150, sequenceid=89, filesize=11.7 K 2024-12-05T11:58:25,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/299a76c94bbd40b3afd80c11fd42acb7 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/299a76c94bbd40b3afd80c11fd42acb7 2024-12-05T11:58:25,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/299a76c94bbd40b3afd80c11fd42acb7, entries=150, sequenceid=89, filesize=11.7 K 2024-12-05T11:58:25,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 517ms, sequenceid=89, compaction requested=false 2024-12-05T11:58:25,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:25,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T11:58:25,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:25,237 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T11:58:25,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/70a474b8932a4d16a60826dc1fe78f9a is 50, key is test_row_0/A:col10/1733399904744/Put/seqid=0 2024-12-05T11:58:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T11:58:25,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742270_1446 (size=12001) 2024-12-05T11:58:25,251 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/70a474b8932a4d16a60826dc1fe78f9a 2024-12-05T11:58:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/9afc208378c94274854a2bbc157df139 is 50, key is test_row_0/B:col10/1733399904744/Put/seqid=0 2024-12-05T11:58:25,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742271_1447 (size=12001) 2024-12-05T11:58:25,364 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:58:25,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:25,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399965423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399965424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399965425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399965537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399965537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399965537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,663 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/9afc208378c94274854a2bbc157df139 2024-12-05T11:58:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/011cf4e372934fb0b12625e5612ab44f is 50, key is test_row_0/C:col10/1733399904744/Put/seqid=0 2024-12-05T11:58:25,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742272_1448 (size=12001) 2024-12-05T11:58:25,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399965747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399965747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399965747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T11:58:26,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399966058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:26,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399966058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:26,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:26,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399966058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:26,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:26,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:26,076 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/011cf4e372934fb0b12625e5612ab44f 2024-12-05T11:58:26,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/70a474b8932a4d16a60826dc1fe78f9a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/70a474b8932a4d16a60826dc1fe78f9a 2024-12-05T11:58:26,085 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/70a474b8932a4d16a60826dc1fe78f9a, entries=150, sequenceid=115, filesize=11.7 K 2024-12-05T11:58:26,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/9afc208378c94274854a2bbc157df139 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/9afc208378c94274854a2bbc157df139 2024-12-05T11:58:26,091 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/9afc208378c94274854a2bbc157df139, entries=150, sequenceid=115, filesize=11.7 K 2024-12-05T11:58:26,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/011cf4e372934fb0b12625e5612ab44f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/011cf4e372934fb0b12625e5612ab44f 2024-12-05T11:58:26,105 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/011cf4e372934fb0b12625e5612ab44f, entries=150, sequenceid=115, filesize=11.7 K 2024-12-05T11:58:26,106 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 869ms, sequenceid=115, compaction requested=true 2024-12-05T11:58:26,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:26,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:26,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-05T11:58:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-05T11:58:26,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-05T11:58:26,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4880 sec 2024-12-05T11:58:26,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.4930 sec 2024-12-05T11:58:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:26,571 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:26,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T11:58:26,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/1bc1c32fff854e34bff7ef61c9d4b450 is 50, key is test_row_0/A:col10/1733399905423/Put/seqid=0 2024-12-05T11:58:26,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742273_1449 (size=14391) 2024-12-05T11:58:26,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/1bc1c32fff854e34bff7ef61c9d4b450 2024-12-05T11:58:26,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/2cd77760f35e4c3995bf08077346f9b8 is 50, key is test_row_0/B:col10/1733399905423/Put/seqid=0 2024-12-05T11:58:26,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742274_1450 (size=12051) 2024-12-05T11:58:26,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399966655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:26,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:26,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399966656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:26,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:26,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399966657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:26,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T11:58:26,768 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:26,769 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-05T11:58:26,770 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:58:26,771 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:26,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:26,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399966777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:26,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399966777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399966777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:58:26,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:58:26,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:26,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:26,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:26,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:26,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:26,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399966988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:26,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/2cd77760f35e4c3995bf08077346f9b8 2024-12-05T11:58:26,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399966988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399966988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:27,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/7fa210b484554d6084c14cc8d5668534 is 50, key is test_row_0/C:col10/1733399905423/Put/seqid=0 2024-12-05T11:58:27,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742275_1451 (size=12051) 2024-12-05T11:58:27,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/7fa210b484554d6084c14cc8d5668534 2024-12-05T11:58:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/1bc1c32fff854e34bff7ef61c9d4b450 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/1bc1c32fff854e34bff7ef61c9d4b450 2024-12-05T11:58:27,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/1bc1c32fff854e34bff7ef61c9d4b450, entries=200, sequenceid=129, filesize=14.1 K 2024-12-05T11:58:27,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/2cd77760f35e4c3995bf08077346f9b8 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/2cd77760f35e4c3995bf08077346f9b8 2024-12-05T11:58:27,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/2cd77760f35e4c3995bf08077346f9b8, entries=150, sequenceid=129, filesize=11.8 K 2024-12-05T11:58:27,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/7fa210b484554d6084c14cc8d5668534 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/7fa210b484554d6084c14cc8d5668534 2024-12-05T11:58:27,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49020 deadline: 1733399967047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:27,053 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in e6ea33c1931a2d534a8c12ecd9fa9d12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:27,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/7fa210b484554d6084c14cc8d5668534, entries=150, sequenceid=129, filesize=11.8 K 2024-12-05T11:58:27,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 484ms, sequenceid=129, compaction requested=true 2024-12-05T11:58:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:27,055 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:27,055 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:27,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:27,057 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52872 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:27,057 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:27,057 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/B is initiating minor compaction (all files) 2024-12-05T11:58:27,057 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/A is initiating minor compaction (all files) 2024-12-05T11:58:27,057 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/A in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,057 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/B in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,057 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/3a6711ddfc8543b38e2ff231df674184, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/a914d4d6ea9a435193aaeaccf691e0d5, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/70a474b8932a4d16a60826dc1fe78f9a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/1bc1c32fff854e34bff7ef61c9d4b450] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=51.6 K 2024-12-05T11:58:27,057 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/02d7dc721607467c9367c9cb50ad4fff, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4e5826ea20f34bbbaa09e6ab7031988f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/9afc208378c94274854a2bbc157df139, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/2cd77760f35e4c3995bf08077346f9b8] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=47.1 K 2024-12-05T11:58:27,057 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 02d7dc721607467c9367c9cb50ad4fff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733399902843 2024-12-05T11:58:27,057 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a6711ddfc8543b38e2ff231df674184, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733399902843 2024-12-05T11:58:27,058 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e5826ea20f34bbbaa09e6ab7031988f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733399904005 2024-12-05T11:58:27,058 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a914d4d6ea9a435193aaeaccf691e0d5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733399904005 2024-12-05T11:58:27,058 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70a474b8932a4d16a60826dc1fe78f9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733399904741 2024-12-05T11:58:27,058 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9afc208378c94274854a2bbc157df139, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733399904741 2024-12-05T11:58:27,058 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bc1c32fff854e34bff7ef61c9d4b450, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733399905417 2024-12-05T11:58:27,058 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd77760f35e4c3995bf08077346f9b8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733399905422 2024-12-05T11:58:27,068 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:27,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T11:58:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:27,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:58:27,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:27,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,081 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#A#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:27,081 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/bec112bfd1cd41e4ba1f07750868ad88 is 50, key is test_row_0/A:col10/1733399905423/Put/seqid=0 2024-12-05T11:58:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:58:27,101 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#B#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:27,101 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/241db7e939264d89978f382d52989f0c is 50, key is test_row_0/B:col10/1733399905423/Put/seqid=0 2024-12-05T11:58:27,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/a9389f738ab24b44bc1d071985c64920 is 50, key is test_row_0/A:col10/1733399907065/Put/seqid=0 2024-12-05T11:58:27,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742276_1452 (size=12325) 2024-12-05T11:58:27,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742277_1453 (size=12325) 2024-12-05T11:58:27,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742278_1454 (size=14541) 2024-12-05T11:58:27,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399967180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:27,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:58:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399967298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:27,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399967308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:27,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399967308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399967308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:27,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:58:27,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:27,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:58:27,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399967508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:27,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:58:27,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:27,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/a9389f738ab24b44bc1d071985c64920 2024-12-05T11:58:27,548 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/bec112bfd1cd41e4ba1f07750868ad88 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/bec112bfd1cd41e4ba1f07750868ad88 2024-12-05T11:58:27,549 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/241db7e939264d89978f382d52989f0c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/241db7e939264d89978f382d52989f0c 2024-12-05T11:58:27,554 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/B of e6ea33c1931a2d534a8c12ecd9fa9d12 into 241db7e939264d89978f382d52989f0c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:27,554 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:27,555 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/B, priority=12, startTime=1733399907055; duration=0sec 2024-12-05T11:58:27,555 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:27,555 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:B 2024-12-05T11:58:27,555 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:27,555 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/A of e6ea33c1931a2d534a8c12ecd9fa9d12 into bec112bfd1cd41e4ba1f07750868ad88(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:27,555 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:27,555 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/A, priority=12, startTime=1733399907055; duration=0sec 2024-12-05T11:58:27,555 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:27,555 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:A 2024-12-05T11:58:27,556 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:27,556 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/C is initiating minor compaction (all files) 2024-12-05T11:58:27,556 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/C in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,556 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/4d04e0866a6548fd935c00c3e6534e75, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/299a76c94bbd40b3afd80c11fd42acb7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/011cf4e372934fb0b12625e5612ab44f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/7fa210b484554d6084c14cc8d5668534] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=47.1 K 2024-12-05T11:58:27,556 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d04e0866a6548fd935c00c3e6534e75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733399902843 2024-12-05T11:58:27,557 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 299a76c94bbd40b3afd80c11fd42acb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733399904005 2024-12-05T11:58:27,557 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 011cf4e372934fb0b12625e5612ab44f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733399904741 2024-12-05T11:58:27,557 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa210b484554d6084c14cc8d5668534, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733399905422 2024-12-05T11:58:27,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/ec28d13c3c5d45e29356e6bad4aff987 is 50, key is test_row_0/B:col10/1733399907065/Put/seqid=0 2024-12-05T11:58:27,570 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#C#compaction#383 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:27,570 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/c7db356762094c7baf64a76719091f7f is 50, key is test_row_0/C:col10/1733399905423/Put/seqid=0 2024-12-05T11:58:27,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742280_1456 (size=12325) 2024-12-05T11:58:27,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742279_1455 (size=12151) 2024-12-05T11:58:27,578 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/c7db356762094c7baf64a76719091f7f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c7db356762094c7baf64a76719091f7f 2024-12-05T11:58:27,582 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/C of e6ea33c1931a2d534a8c12ecd9fa9d12 into c7db356762094c7baf64a76719091f7f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:27,582 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:27,582 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/C, priority=12, startTime=1733399907055; duration=0sec 2024-12-05T11:58:27,582 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:27,582 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:C 2024-12-05T11:58:27,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:58:27,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:27,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399967817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:27,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399967818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:27,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399967828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:27,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399967828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:27,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:58:27,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:27,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:27,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:58:27,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/ec28d13c3c5d45e29356e6bad4aff987 2024-12-05T11:58:27,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/18cd4ecbc24044ce848de08072532a7f is 50, key is test_row_0/C:col10/1733399907065/Put/seqid=0 2024-12-05T11:58:27,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742281_1457 (size=12151) 2024-12-05T11:58:28,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:58:28,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:28,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:28,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:28,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:28,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:58:28,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:28,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:28,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:28,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:28,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:28,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:58:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:28,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:28,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399968347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:28,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/18cd4ecbc24044ce848de08072532a7f 2024-12-05T11:58:28,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/a9389f738ab24b44bc1d071985c64920 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/a9389f738ab24b44bc1d071985c64920 2024-12-05T11:58:28,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/a9389f738ab24b44bc1d071985c64920, entries=200, sequenceid=152, filesize=14.2 K 2024-12-05T11:58:28,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/ec28d13c3c5d45e29356e6bad4aff987 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/ec28d13c3c5d45e29356e6bad4aff987 2024-12-05T11:58:28,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/ec28d13c3c5d45e29356e6bad4aff987, entries=150, sequenceid=152, filesize=11.9 K 2024-12-05T11:58:28,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/18cd4ecbc24044ce848de08072532a7f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/18cd4ecbc24044ce848de08072532a7f 2024-12-05T11:58:28,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/18cd4ecbc24044ce848de08072532a7f, entries=150, sequenceid=152, filesize=11.9 K 2024-12-05T11:58:28,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 1333ms, sequenceid=152, compaction requested=false 2024-12-05T11:58:28,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:28,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:58:28,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:28,459 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T11:58:28,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/34a46a3a78c44c74a9b17f4b21ac2ac1 is 50, key is test_row_0/A:col10/1733399907143/Put/seqid=0 2024-12-05T11:58:28,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742282_1458 (size=12151) 2024-12-05T11:58:28,495 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/34a46a3a78c44c74a9b17f4b21ac2ac1 2024-12-05T11:58:28,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/07c7808eb4534e6c9a8bf5f6405b5c3c is 50, key is test_row_0/B:col10/1733399907143/Put/seqid=0 2024-12-05T11:58:28,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742283_1459 (size=12151) 2024-12-05T11:58:28,532 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/07c7808eb4534e6c9a8bf5f6405b5c3c 2024-12-05T11:58:28,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/47522f00999a49c0802ff77513289e54 is 50, key is test_row_0/C:col10/1733399907143/Put/seqid=0 2024-12-05T11:58:28,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742284_1460 (size=12151) 2024-12-05T11:58:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:28,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399968888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:28,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399968889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:28,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399968896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:28,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:28,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:28,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:58:28,945 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/47522f00999a49c0802ff77513289e54 2024-12-05T11:58:28,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/34a46a3a78c44c74a9b17f4b21ac2ac1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/34a46a3a78c44c74a9b17f4b21ac2ac1 2024-12-05T11:58:28,953 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/34a46a3a78c44c74a9b17f4b21ac2ac1, entries=150, sequenceid=168, filesize=11.9 K 2024-12-05T11:58:28,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/07c7808eb4534e6c9a8bf5f6405b5c3c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/07c7808eb4534e6c9a8bf5f6405b5c3c 2024-12-05T11:58:28,956 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/07c7808eb4534e6c9a8bf5f6405b5c3c, entries=150, sequenceid=168, filesize=11.9 K 2024-12-05T11:58:28,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/47522f00999a49c0802ff77513289e54 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/47522f00999a49c0802ff77513289e54 2024-12-05T11:58:28,960 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/47522f00999a49c0802ff77513289e54, entries=150, sequenceid=168, filesize=11.9 K 2024-12-05T11:58:28,961 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 502ms, sequenceid=168, compaction requested=true 2024-12-05T11:58:28,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:28,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:28,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-05T11:58:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-05T11:58:28,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-05T11:58:28,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1920 sec 2024-12-05T11:58:28,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.1960 sec 2024-12-05T11:58:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:29,007 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:29,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T11:58:29,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/bc0056d873454790a56a17632595be0e is 50, key is test_row_0/A:col10/1733399908887/Put/seqid=0 2024-12-05T11:58:29,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742285_1461 (size=16931) 2024-12-05T11:58:29,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/bc0056d873454790a56a17632595be0e 2024-12-05T11:58:29,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/e66cb2a317074f84a12d73f94da0fbf0 is 50, key is test_row_0/B:col10/1733399908887/Put/seqid=0 2024-12-05T11:58:29,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742286_1462 (size=12151) 2024-12-05T11:58:29,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399969033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399969037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399969037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399969147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399969158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399969158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399969358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399969358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,363 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_1' in e6ea33c1931a2d534a8c12ecd9fa9d12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399969368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399969368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/e66cb2a317074f84a12d73f94da0fbf0 2024-12-05T11:58:29,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/d94f418edd1d4239bd11c442693650d2 is 50, key is test_row_0/C:col10/1733399908887/Put/seqid=0 2024-12-05T11:58:29,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742287_1463 (size=12151) 2024-12-05T11:58:29,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399969668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399969678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399969678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:29,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/d94f418edd1d4239bd11c442693650d2 2024-12-05T11:58:29,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/bc0056d873454790a56a17632595be0e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/bc0056d873454790a56a17632595be0e 2024-12-05T11:58:29,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/bc0056d873454790a56a17632595be0e, entries=250, sequenceid=193, filesize=16.5 K 2024-12-05T11:58:29,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/e66cb2a317074f84a12d73f94da0fbf0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/e66cb2a317074f84a12d73f94da0fbf0 2024-12-05T11:58:29,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/e66cb2a317074f84a12d73f94da0fbf0, entries=150, sequenceid=193, filesize=11.9 K 2024-12-05T11:58:29,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/d94f418edd1d4239bd11c442693650d2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/d94f418edd1d4239bd11c442693650d2 2024-12-05T11:58:29,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/d94f418edd1d4239bd11c442693650d2, entries=150, sequenceid=193, filesize=11.9 K 2024-12-05T11:58:29,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 863ms, sequenceid=193, compaction requested=true 2024-12-05T11:58:29,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:29,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:29,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:29,871 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:29,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:29,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:29,871 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:29,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:29,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:29,872 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48778 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:29,872 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55948 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:29,872 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/B is initiating minor compaction (all files) 2024-12-05T11:58:29,872 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/A is initiating minor compaction (all files) 2024-12-05T11:58:29,872 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/A in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:29,872 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/B in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:29,872 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/bec112bfd1cd41e4ba1f07750868ad88, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/a9389f738ab24b44bc1d071985c64920, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/34a46a3a78c44c74a9b17f4b21ac2ac1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/bc0056d873454790a56a17632595be0e] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=54.6 K 2024-12-05T11:58:29,872 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/241db7e939264d89978f382d52989f0c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/ec28d13c3c5d45e29356e6bad4aff987, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/07c7808eb4534e6c9a8bf5f6405b5c3c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/e66cb2a317074f84a12d73f94da0fbf0] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=47.6 K 2024-12-05T11:58:29,872 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting bec112bfd1cd41e4ba1f07750868ad88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733399905422 2024-12-05T11:58:29,872 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 241db7e939264d89978f382d52989f0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733399905422 2024-12-05T11:58:29,872 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9389f738ab24b44bc1d071985c64920, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733399906632 2024-12-05T11:58:29,872 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ec28d13c3c5d45e29356e6bad4aff987, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733399906653 2024-12-05T11:58:29,872 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34a46a3a78c44c74a9b17f4b21ac2ac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733399907143 2024-12-05T11:58:29,872 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 07c7808eb4534e6c9a8bf5f6405b5c3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733399907143 2024-12-05T11:58:29,873 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc0056d873454790a56a17632595be0e, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733399908877 2024-12-05T11:58:29,873 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting e66cb2a317074f84a12d73f94da0fbf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733399908877 2024-12-05T11:58:29,884 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#B#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:29,884 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/fae163aca91244748a73d15f1edb085f is 50, key is test_row_0/B:col10/1733399908887/Put/seqid=0 2024-12-05T11:58:29,885 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#A#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:29,886 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/7a159e4b47a5493dbe53e58e657abe68 is 50, key is test_row_0/A:col10/1733399908887/Put/seqid=0 2024-12-05T11:58:29,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742288_1464 (size=12561) 2024-12-05T11:58:29,913 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/fae163aca91244748a73d15f1edb085f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/fae163aca91244748a73d15f1edb085f 2024-12-05T11:58:29,918 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/B of e6ea33c1931a2d534a8c12ecd9fa9d12 into fae163aca91244748a73d15f1edb085f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:29,918 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:29,918 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/B, priority=12, startTime=1733399909871; duration=0sec 2024-12-05T11:58:29,918 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:29,918 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:B 2024-12-05T11:58:29,918 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:29,919 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48778 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:29,919 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/C is initiating minor compaction (all files) 2024-12-05T11:58:29,919 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/C in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:29,919 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c7db356762094c7baf64a76719091f7f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/18cd4ecbc24044ce848de08072532a7f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/47522f00999a49c0802ff77513289e54, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/d94f418edd1d4239bd11c442693650d2] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=47.6 K 2024-12-05T11:58:29,920 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c7db356762094c7baf64a76719091f7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733399905422 2024-12-05T11:58:29,920 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 18cd4ecbc24044ce848de08072532a7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733399906653 2024-12-05T11:58:29,922 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 47522f00999a49c0802ff77513289e54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733399907143 2024-12-05T11:58:29,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742289_1465 (size=12561) 2024-12-05T11:58:29,923 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d94f418edd1d4239bd11c442693650d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733399908877 2024-12-05T11:58:29,930 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/7a159e4b47a5493dbe53e58e657abe68 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7a159e4b47a5493dbe53e58e657abe68 2024-12-05T11:58:29,935 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/A of e6ea33c1931a2d534a8c12ecd9fa9d12 into 7a159e4b47a5493dbe53e58e657abe68(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:29,935 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:29,935 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/A, priority=12, startTime=1733399909871; duration=0sec 2024-12-05T11:58:29,935 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:29,935 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:A 2024-12-05T11:58:29,937 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#C#compaction#393 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:29,937 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/6da0178c59f143dc9ea8e118d7d6dbf8 is 50, key is test_row_0/C:col10/1733399908887/Put/seqid=0 2024-12-05T11:58:29,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742290_1466 (size=12561) 2024-12-05T11:58:29,946 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/6da0178c59f143dc9ea8e118d7d6dbf8 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/6da0178c59f143dc9ea8e118d7d6dbf8 2024-12-05T11:58:29,951 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/C of e6ea33c1931a2d534a8c12ecd9fa9d12 into 6da0178c59f143dc9ea8e118d7d6dbf8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:29,951 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:29,951 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/C, priority=12, startTime=1733399909871; duration=0sec 2024-12-05T11:58:29,951 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:29,951 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:C 2024-12-05T11:58:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:30,180 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:30,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T11:58:30,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/eabb2b4b853d44cc9962fca32f947dd6 is 50, key is test_row_0/A:col10/1733399909026/Put/seqid=0 2024-12-05T11:58:30,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742291_1467 (size=14541) 2024-12-05T11:58:30,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399970229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:30,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:30,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399970231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:30,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399970232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:30,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399970337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:30,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399970338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:30,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399970348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:30,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399970547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:30,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399970548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:30,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399970558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:30,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/eabb2b4b853d44cc9962fca32f947dd6 2024-12-05T11:58:30,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/84d7f62d4cf04fbb9566f9afbb6b4b30 is 50, key is test_row_0/B:col10/1733399909026/Put/seqid=0 2024-12-05T11:58:30,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742292_1468 (size=12151) 2024-12-05T11:58:30,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399970857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:30,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399970857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:30,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399970868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:58:30,948 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:30,948 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-05T11:58:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:58:30,950 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:30,950 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:30,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:31,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/84d7f62d4cf04fbb9566f9afbb6b4b30 2024-12-05T11:58:31,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/3189a2423203407394311c50e9263b09 is 50, key is test_row_0/C:col10/1733399909026/Put/seqid=0 2024-12-05T11:58:31,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742293_1469 (size=12151) 2024-12-05T11:58:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:58:31,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49020 deadline: 1733399971078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:31,082 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in e6ea33c1931a2d534a8c12ecd9fa9d12 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:31,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T11:58:31,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:31,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:31,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:31,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:31,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:31,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T11:58:31,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:31,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:31,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:31,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:31,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:58:31,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399971368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:31,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399971368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:31,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:31,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399971378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:31,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399971387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:31,390 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_1' in e6ea33c1931a2d534a8c12ecd9fa9d12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:31,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T11:58:31,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:31,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:31,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:31,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:31,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:31,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/3189a2423203407394311c50e9263b09 2024-12-05T11:58:31,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/eabb2b4b853d44cc9962fca32f947dd6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/eabb2b4b853d44cc9962fca32f947dd6 2024-12-05T11:58:31,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/eabb2b4b853d44cc9962fca32f947dd6, entries=200, sequenceid=209, filesize=14.2 K 2024-12-05T11:58:31,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/84d7f62d4cf04fbb9566f9afbb6b4b30 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/84d7f62d4cf04fbb9566f9afbb6b4b30 2024-12-05T11:58:31,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/84d7f62d4cf04fbb9566f9afbb6b4b30, entries=150, sequenceid=209, filesize=11.9 K 2024-12-05T11:58:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/3189a2423203407394311c50e9263b09 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/3189a2423203407394311c50e9263b09 2024-12-05T11:58:31,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/3189a2423203407394311c50e9263b09, entries=150, sequenceid=209, filesize=11.9 K 2024-12-05T11:58:31,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 1265ms, sequenceid=209, compaction requested=false 2024-12-05T11:58:31,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:31,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T11:58:31,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:31,560 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T11:58:31,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/0718b02d1b2749e489adbbda4c4f1e94 is 50, key is test_row_0/A:col10/1733399910228/Put/seqid=0 2024-12-05T11:58:31,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742294_1470 (size=12151) 2024-12-05T11:58:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:58:31,968 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/0718b02d1b2749e489adbbda4c4f1e94 2024-12-05T11:58:31,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/aced7d0ff7684aefa7579c35d7c4b5dd is 50, key is test_row_0/B:col10/1733399910228/Put/seqid=0 2024-12-05T11:58:31,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742295_1471 (size=12151) 2024-12-05T11:58:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:58:32,378 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/aced7d0ff7684aefa7579c35d7c4b5dd 2024-12-05T11:58:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:32,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:32,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/242749f685944504926af688c84c2bc3 is 50, key is test_row_0/C:col10/1733399910228/Put/seqid=0 2024-12-05T11:58:32,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742296_1472 (size=12151) 2024-12-05T11:58:32,403 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/242749f685944504926af688c84c2bc3 2024-12-05T11:58:32,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/0718b02d1b2749e489adbbda4c4f1e94 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/0718b02d1b2749e489adbbda4c4f1e94 2024-12-05T11:58:32,412 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/0718b02d1b2749e489adbbda4c4f1e94, entries=150, sequenceid=232, filesize=11.9 K 2024-12-05T11:58:32,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/aced7d0ff7684aefa7579c35d7c4b5dd as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/aced7d0ff7684aefa7579c35d7c4b5dd 2024-12-05T11:58:32,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399972407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:32,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399972409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:32,417 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/aced7d0ff7684aefa7579c35d7c4b5dd, entries=150, sequenceid=232, filesize=11.9 K 2024-12-05T11:58:32,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/242749f685944504926af688c84c2bc3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/242749f685944504926af688c84c2bc3 2024-12-05T11:58:32,421 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/242749f685944504926af688c84c2bc3, entries=150, sequenceid=232, filesize=11.9 K 2024-12-05T11:58:32,422 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 862ms, sequenceid=232, compaction requested=true 2024-12-05T11:58:32,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:32,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:32,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-05T11:58:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-05T11:58:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:32,424 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:32,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T11:58:32,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-05T11:58:32,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4730 sec 2024-12-05T11:58:32,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4790 sec 2024-12-05T11:58:32,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/ded38006a2b6449ea9f419fff373be93 is 50, key is test_row_0/A:col10/1733399912413/Put/seqid=0 2024-12-05T11:58:32,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742297_1473 (size=14541) 2024-12-05T11:58:32,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399972524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:32,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399972528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:32,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399972529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:32,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399972637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:32,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:32,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399972638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:32,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:32,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:32,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399972738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:32,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:32,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/ded38006a2b6449ea9f419fff373be93 2024-12-05T11:58:32,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/b52d0a9ff58d4ab0a79fc6909ab6fcca is 50, key is test_row_0/B:col10/1733399912413/Put/seqid=0 2024-12-05T11:58:32,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:32,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399972848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:32,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399972848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:32,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742298_1474 (size=12151) 2024-12-05T11:58:32,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/b52d0a9ff58d4ab0a79fc6909ab6fcca 2024-12-05T11:58:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:32,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/b6fa15bd5d5e458493a501d370c89768 is 50, key is test_row_0/C:col10/1733399912413/Put/seqid=0 2024-12-05T11:58:32,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742299_1475 (size=12151) 2024-12-05T11:58:33,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399973047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:58:33,108 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:33,109 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-05T11:58:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T11:58:33,110 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:33,111 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:33,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:33,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399973157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:33,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:33,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399973168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T11:58:33,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T11:58:33,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:33,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:33,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:33,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:33,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:33,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/b6fa15bd5d5e458493a501d370c89768 2024-12-05T11:58:33,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/ded38006a2b6449ea9f419fff373be93 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ded38006a2b6449ea9f419fff373be93 2024-12-05T11:58:33,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ded38006a2b6449ea9f419fff373be93, entries=200, sequenceid=246, filesize=14.2 K 2024-12-05T11:58:33,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/b52d0a9ff58d4ab0a79fc6909ab6fcca as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/b52d0a9ff58d4ab0a79fc6909ab6fcca 2024-12-05T11:58:33,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/b52d0a9ff58d4ab0a79fc6909ab6fcca, entries=150, sequenceid=246, filesize=11.9 K 2024-12-05T11:58:33,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/b6fa15bd5d5e458493a501d370c89768 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/b6fa15bd5d5e458493a501d370c89768 2024-12-05T11:58:33,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/b6fa15bd5d5e458493a501d370c89768, entries=150, sequenceid=246, filesize=11.9 K 2024-12-05T11:58:33,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 869ms, sequenceid=246, compaction requested=true 2024-12-05T11:58:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:33,293 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T11:58:33,293 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:33,294 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:33,294 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:33,294 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/B is initiating minor compaction (all files) 2024-12-05T11:58:33,294 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/A is initiating minor compaction (all files) 2024-12-05T11:58:33,294 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/B in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:33,294 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/A in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:33,294 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/fae163aca91244748a73d15f1edb085f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/84d7f62d4cf04fbb9566f9afbb6b4b30, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/aced7d0ff7684aefa7579c35d7c4b5dd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/b52d0a9ff58d4ab0a79fc6909ab6fcca] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=47.9 K 2024-12-05T11:58:33,294 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7a159e4b47a5493dbe53e58e657abe68, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/eabb2b4b853d44cc9962fca32f947dd6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/0718b02d1b2749e489adbbda4c4f1e94, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ded38006a2b6449ea9f419fff373be93] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=52.5 K 2024-12-05T11:58:33,295 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a159e4b47a5493dbe53e58e657abe68, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733399908877 2024-12-05T11:58:33,295 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting fae163aca91244748a73d15f1edb085f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733399908877 2024-12-05T11:58:33,296 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting eabb2b4b853d44cc9962fca32f947dd6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733399909015 2024-12-05T11:58:33,296 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 84d7f62d4cf04fbb9566f9afbb6b4b30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733399909026 2024-12-05T11:58:33,296 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0718b02d1b2749e489adbbda4c4f1e94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733399910222 2024-12-05T11:58:33,296 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting aced7d0ff7684aefa7579c35d7c4b5dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733399910222 2024-12-05T11:58:33,297 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ded38006a2b6449ea9f419fff373be93, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733399912392 2024-12-05T11:58:33,297 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting b52d0a9ff58d4ab0a79fc6909ab6fcca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733399912392 2024-12-05T11:58:33,308 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#A#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:33,308 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/1bf84ad90b4f43e18bba9a865226fa0c is 50, key is test_row_0/A:col10/1733399912413/Put/seqid=0 2024-12-05T11:58:33,310 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#B#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:33,310 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/cd4e9be06b09465da199cf12b5bb73dd is 50, key is test_row_0/B:col10/1733399912413/Put/seqid=0 2024-12-05T11:58:33,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742300_1476 (size=12697) 2024-12-05T11:58:33,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742301_1477 (size=12697) 2024-12-05T11:58:33,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T11:58:33,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:33,420 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-05T11:58:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T11:58:33,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/9587c73c9a8a4750b66fb66898fef2f3 is 50, key is test_row_0/A:col10/1733399912528/Put/seqid=0 2024-12-05T11:58:33,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742302_1478 (size=12301) 2024-12-05T11:58:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:33,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:33,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399973691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:33,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399973695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:33,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399973699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:33,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:33,730 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/1bf84ad90b4f43e18bba9a865226fa0c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/1bf84ad90b4f43e18bba9a865226fa0c 2024-12-05T11:58:33,731 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/cd4e9be06b09465da199cf12b5bb73dd as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/cd4e9be06b09465da199cf12b5bb73dd 2024-12-05T11:58:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T11:58:33,750 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/A of e6ea33c1931a2d534a8c12ecd9fa9d12 into 1bf84ad90b4f43e18bba9a865226fa0c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:33,750 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:33,750 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/A, priority=12, startTime=1733399913293; duration=0sec 2024-12-05T11:58:33,750 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:33,750 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:A 2024-12-05T11:58:33,750 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:33,753 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/B of e6ea33c1931a2d534a8c12ecd9fa9d12 into cd4e9be06b09465da199cf12b5bb73dd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:33,754 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:33,754 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/B, priority=12, startTime=1733399913293; duration=0sec 2024-12-05T11:58:33,754 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:33,754 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:B 2024-12-05T11:58:33,759 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:33,759 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/C is initiating minor compaction (all files) 2024-12-05T11:58:33,759 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/C in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:33,759 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/6da0178c59f143dc9ea8e118d7d6dbf8, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/3189a2423203407394311c50e9263b09, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/242749f685944504926af688c84c2bc3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/b6fa15bd5d5e458493a501d370c89768] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=47.9 K 2024-12-05T11:58:33,760 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6da0178c59f143dc9ea8e118d7d6dbf8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733399908877 2024-12-05T11:58:33,761 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3189a2423203407394311c50e9263b09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733399909026 2024-12-05T11:58:33,761 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 242749f685944504926af688c84c2bc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733399910222 2024-12-05T11:58:33,762 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6fa15bd5d5e458493a501d370c89768, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733399912392 2024-12-05T11:58:33,801 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#C#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:33,802 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/cbd63a6c82ac4146986656bd65186194 is 50, key is test_row_0/C:col10/1733399912413/Put/seqid=0 2024-12-05T11:58:33,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399973818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:33,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399973819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:33,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399973820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:33,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742303_1479 (size=12697) 2024-12-05T11:58:33,885 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/9587c73c9a8a4750b66fb66898fef2f3 2024-12-05T11:58:33,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/6853978e232b4e288bb0537e790c81a5 is 50, key is test_row_0/B:col10/1733399912528/Put/seqid=0 2024-12-05T11:58:33,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742304_1480 (size=12301) 2024-12-05T11:58:34,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399974038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:34,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399974038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:34,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399974038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T11:58:34,253 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/cbd63a6c82ac4146986656bd65186194 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/cbd63a6c82ac4146986656bd65186194 2024-12-05T11:58:34,261 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/C of e6ea33c1931a2d534a8c12ecd9fa9d12 into cbd63a6c82ac4146986656bd65186194(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:34,261 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:34,261 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/C, priority=12, startTime=1733399913293; duration=0sec 2024-12-05T11:58:34,261 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:34,261 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:C 2024-12-05T11:58:34,336 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/6853978e232b4e288bb0537e790c81a5 2024-12-05T11:58:34,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:34,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399974360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:34,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/1e3f1c300fce421a8afd903790bd584b is 50, key is test_row_0/C:col10/1733399912528/Put/seqid=0 2024-12-05T11:58:34,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399974360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:34,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:34,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399974361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:34,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742305_1481 (size=12301) 2024-12-05T11:58:34,797 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/1e3f1c300fce421a8afd903790bd584b 2024-12-05T11:58:34,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/9587c73c9a8a4750b66fb66898fef2f3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/9587c73c9a8a4750b66fb66898fef2f3 2024-12-05T11:58:34,805 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/9587c73c9a8a4750b66fb66898fef2f3, entries=150, sequenceid=268, filesize=12.0 K 2024-12-05T11:58:34,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/6853978e232b4e288bb0537e790c81a5 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/6853978e232b4e288bb0537e790c81a5 2024-12-05T11:58:34,817 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/6853978e232b4e288bb0537e790c81a5, entries=150, sequenceid=268, filesize=12.0 K 2024-12-05T11:58:34,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/1e3f1c300fce421a8afd903790bd584b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1e3f1c300fce421a8afd903790bd584b 2024-12-05T11:58:34,823 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1e3f1c300fce421a8afd903790bd584b, entries=150, sequenceid=268, filesize=12.0 K 2024-12-05T11:58:34,824 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 1404ms, sequenceid=268, compaction requested=false 2024-12-05T11:58:34,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:34,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:34,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-05T11:58:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-05T11:58:34,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-05T11:58:34,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7150 sec 2024-12-05T11:58:34,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7190 sec 2024-12-05T11:58:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:34,887 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:34,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-05T11:58:34,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/2435d33c1ed343f88aae916475bdf258 is 50, key is test_row_0/A:col10/1733399914871/Put/seqid=0 2024-12-05T11:58:34,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742306_1482 (size=14741) 2024-12-05T11:58:34,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/2435d33c1ed343f88aae916475bdf258 2024-12-05T11:58:34,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/4c31a54d3411404c8ce57f3dc33cbed8 is 50, key is test_row_0/B:col10/1733399914871/Put/seqid=0 2024-12-05T11:58:35,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399974990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742307_1483 (size=12301) 2024-12-05T11:58:35,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399974998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399974998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399975108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399975128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399975129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T11:58:35,258 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:35,259 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-05T11:58:35,261 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T11:58:35,262 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:35,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:35,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399975328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399975338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399975348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T11:58:35,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T11:58:35,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:35,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:35,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:35,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:35,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/4c31a54d3411404c8ce57f3dc33cbed8 2024-12-05T11:58:35,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/c0d8bae7277d43ff9a707903b425bb1d is 50, key is test_row_0/C:col10/1733399914871/Put/seqid=0 2024-12-05T11:58:35,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49072 deadline: 1733399975418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,433 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_1' in e6ea33c1931a2d534a8c12ecd9fa9d12 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:35,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742308_1484 (size=12301) 2024-12-05T11:58:35,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T11:58:35,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:35,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:35,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:35,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:35,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T11:58:35,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399975648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399975658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399975668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:35,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:35,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:35,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T11:58:35,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:35,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:35,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:35,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:35,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:35,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T11:58:35,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:35,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:35,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:35,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:35,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T11:58:35,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/c0d8bae7277d43ff9a707903b425bb1d 2024-12-05T11:58:35,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/2435d33c1ed343f88aae916475bdf258 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/2435d33c1ed343f88aae916475bdf258 2024-12-05T11:58:35,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/2435d33c1ed343f88aae916475bdf258, entries=200, sequenceid=286, filesize=14.4 K 2024-12-05T11:58:35,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/4c31a54d3411404c8ce57f3dc33cbed8 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4c31a54d3411404c8ce57f3dc33cbed8 2024-12-05T11:58:35,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4c31a54d3411404c8ce57f3dc33cbed8, entries=150, sequenceid=286, filesize=12.0 K 2024-12-05T11:58:35,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/c0d8bae7277d43ff9a707903b425bb1d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c0d8bae7277d43ff9a707903b425bb1d 2024-12-05T11:58:35,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c0d8bae7277d43ff9a707903b425bb1d, entries=150, sequenceid=286, filesize=12.0 K 2024-12-05T11:58:35,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 1045ms, sequenceid=286, compaction requested=true 2024-12-05T11:58:35,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:35,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:35,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:35,933 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:35,933 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:35,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:35,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:35,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:35,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:35,934 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:35,934 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/B is initiating minor compaction (all files) 2024-12-05T11:58:35,934 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/B in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:35,934 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/cd4e9be06b09465da199cf12b5bb73dd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/6853978e232b4e288bb0537e790c81a5, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4c31a54d3411404c8ce57f3dc33cbed8] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=36.4 K 2024-12-05T11:58:35,934 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:35,934 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/A is initiating minor compaction (all files) 2024-12-05T11:58:35,934 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/A in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:35,934 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting cd4e9be06b09465da199cf12b5bb73dd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733399912392 2024-12-05T11:58:35,934 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/1bf84ad90b4f43e18bba9a865226fa0c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/9587c73c9a8a4750b66fb66898fef2f3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/2435d33c1ed343f88aae916475bdf258] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=38.8 K 2024-12-05T11:58:35,934 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6853978e232b4e288bb0537e790c81a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733399912513 2024-12-05T11:58:35,934 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bf84ad90b4f43e18bba9a865226fa0c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733399912392 2024-12-05T11:58:35,935 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c31a54d3411404c8ce57f3dc33cbed8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733399913673 2024-12-05T11:58:35,935 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9587c73c9a8a4750b66fb66898fef2f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733399912513 2024-12-05T11:58:35,935 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2435d33c1ed343f88aae916475bdf258, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733399913673 2024-12-05T11:58:35,960 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#A#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:35,961 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/f985cb71c6e5464b835176aa3cfcede0 is 50, key is test_row_0/A:col10/1733399914871/Put/seqid=0 2024-12-05T11:58:35,974 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#B#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:35,974 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/27262a7e4ed842b792fc649a34cb5620 is 50, key is test_row_0/B:col10/1733399914871/Put/seqid=0 2024-12-05T11:58:36,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742309_1485 (size=12949) 2024-12-05T11:58:36,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T11:58:36,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:36,031 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-05T11:58:36,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742310_1486 (size=12949) 2024-12-05T11:58:36,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/ad38275ac3b44479adbaf585596bb639 is 50, key is test_row_0/A:col10/1733399914979/Put/seqid=0 2024-12-05T11:58:36,054 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/27262a7e4ed842b792fc649a34cb5620 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/27262a7e4ed842b792fc649a34cb5620 2024-12-05T11:58:36,058 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/B of e6ea33c1931a2d534a8c12ecd9fa9d12 into 27262a7e4ed842b792fc649a34cb5620(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:36,059 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:36,059 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/B, priority=13, startTime=1733399915933; duration=0sec 2024-12-05T11:58:36,059 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:36,059 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:B 2024-12-05T11:58:36,059 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:36,060 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:36,060 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/C is initiating minor compaction (all files) 2024-12-05T11:58:36,060 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/C in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:36,060 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/cbd63a6c82ac4146986656bd65186194, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1e3f1c300fce421a8afd903790bd584b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c0d8bae7277d43ff9a707903b425bb1d] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=36.4 K 2024-12-05T11:58:36,060 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting cbd63a6c82ac4146986656bd65186194, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733399912392 2024-12-05T11:58:36,061 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e3f1c300fce421a8afd903790bd584b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733399912513 2024-12-05T11:58:36,061 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d8bae7277d43ff9a707903b425bb1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733399913673 2024-12-05T11:58:36,082 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#C#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:36,082 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/437857e0cecf4ccc9c1bde30f6d7b557 is 50, key is test_row_0/C:col10/1733399914871/Put/seqid=0 2024-12-05T11:58:36,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742311_1487 (size=12301) 2024-12-05T11:58:36,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742312_1488 (size=12949) 2024-12-05T11:58:36,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:36,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399976237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:36,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399976238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:36,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399976253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:36,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:36,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399976358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:36,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399976368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399976368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T11:58:36,432 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/f985cb71c6e5464b835176aa3cfcede0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/f985cb71c6e5464b835176aa3cfcede0 2024-12-05T11:58:36,438 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/A of e6ea33c1931a2d534a8c12ecd9fa9d12 into f985cb71c6e5464b835176aa3cfcede0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:36,438 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:36,438 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/A, priority=13, startTime=1733399915933; duration=0sec 2024-12-05T11:58:36,438 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:36,438 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:A 2024-12-05T11:58:36,525 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/ad38275ac3b44479adbaf585596bb639 2024-12-05T11:58:36,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/558c560e57b44ed283283a0ee3c8acec is 50, key is test_row_0/B:col10/1733399914979/Put/seqid=0 2024-12-05T11:58:36,571 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/437857e0cecf4ccc9c1bde30f6d7b557 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/437857e0cecf4ccc9c1bde30f6d7b557 2024-12-05T11:58:36,577 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/C of e6ea33c1931a2d534a8c12ecd9fa9d12 into 437857e0cecf4ccc9c1bde30f6d7b557(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:36,577 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:36,577 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/C, priority=13, startTime=1733399915933; duration=0sec 2024-12-05T11:58:36,577 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:36,577 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:C 2024-12-05T11:58:36,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742313_1489 (size=12301) 2024-12-05T11:58:36,583 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/558c560e57b44ed283283a0ee3c8acec 2024-12-05T11:58:36,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399976578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:36,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399976578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:36,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399976578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:36,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/1fc15a7900d54725aaabc3990ee90765 is 50, key is test_row_0/C:col10/1733399914979/Put/seqid=0 2024-12-05T11:58:36,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742314_1490 (size=12301) 2024-12-05T11:58:36,636 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/1fc15a7900d54725aaabc3990ee90765 2024-12-05T11:58:36,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/ad38275ac3b44479adbaf585596bb639 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ad38275ac3b44479adbaf585596bb639 2024-12-05T11:58:36,653 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ad38275ac3b44479adbaf585596bb639, entries=150, sequenceid=308, filesize=12.0 K 2024-12-05T11:58:36,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/558c560e57b44ed283283a0ee3c8acec as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/558c560e57b44ed283283a0ee3c8acec 2024-12-05T11:58:36,660 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/558c560e57b44ed283283a0ee3c8acec, entries=150, sequenceid=308, filesize=12.0 K 2024-12-05T11:58:36,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/1fc15a7900d54725aaabc3990ee90765 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1fc15a7900d54725aaabc3990ee90765 2024-12-05T11:58:36,668 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1fc15a7900d54725aaabc3990ee90765, entries=150, sequenceid=308, filesize=12.0 K 2024-12-05T11:58:36,669 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 638ms, sequenceid=308, compaction requested=false 2024-12-05T11:58:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-05T11:58:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-05T11:58:36,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-05T11:58:36,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-12-05T11:58:36,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4150 sec 2024-12-05T11:58:36,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:36,919 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:36,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-05T11:58:36,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/7869087b720748d9a0eb0952ddc96779 is 50, key is test_row_0/A:col10/1733399916915/Put/seqid=0 2024-12-05T11:58:36,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742315_1491 (size=12301) 2024-12-05T11:58:37,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:37,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399977019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:37,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399977026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:37,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399977032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:37,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:37,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399977150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399977150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399977158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:37,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399977369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:37,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399977369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:37,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399977377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:37,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/7869087b720748d9a0eb0952ddc96779 2024-12-05T11:58:37,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/6831e4bd1ef447588d9fdcc5f0d1c524 is 50, key is test_row_0/B:col10/1733399916915/Put/seqid=0 2024-12-05T11:58:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T11:58:37,409 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:37,409 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-05T11:58:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T11:58:37,413 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:37,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:37,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:37,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742316_1492 (size=12301) 2024-12-05T11:58:37,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/6831e4bd1ef447588d9fdcc5f0d1c524 2024-12-05T11:58:37,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/4194344bab2847f9b13efd548ce95ff5 is 50, key is test_row_0/C:col10/1733399916915/Put/seqid=0 2024-12-05T11:58:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T11:58:37,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742317_1493 (size=12301) 2024-12-05T11:58:37,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T11:58:37,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:37,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:37,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:37,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:37,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:37,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399977688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399977688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:37,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399977697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T11:58:37,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T11:58:37,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:37,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:37,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:37,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:37,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:37,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T11:58:37,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:37,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:37,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:37,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:37,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:37,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/4194344bab2847f9b13efd548ce95ff5 2024-12-05T11:58:37,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/7869087b720748d9a0eb0952ddc96779 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7869087b720748d9a0eb0952ddc96779 2024-12-05T11:58:37,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7869087b720748d9a0eb0952ddc96779, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T11:58:37,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/6831e4bd1ef447588d9fdcc5f0d1c524 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/6831e4bd1ef447588d9fdcc5f0d1c524 2024-12-05T11:58:37,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/6831e4bd1ef447588d9fdcc5f0d1c524, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T11:58:37,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/4194344bab2847f9b13efd548ce95ff5 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/4194344bab2847f9b13efd548ce95ff5 2024-12-05T11:58:37,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/4194344bab2847f9b13efd548ce95ff5, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T11:58:37,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 1061ms, sequenceid=328, compaction requested=true 2024-12-05T11:58:37,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:37,981 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:37,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:37,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:37,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:37,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:37,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:37,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T11:58:37,981 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:37,982 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/A is initiating minor compaction (all files) 2024-12-05T11:58:37,982 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/A in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:37,982 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/f985cb71c6e5464b835176aa3cfcede0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ad38275ac3b44479adbaf585596bb639, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7869087b720748d9a0eb0952ddc96779] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=36.7 K 2024-12-05T11:58:37,982 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:37,982 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f985cb71c6e5464b835176aa3cfcede0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733399913673 2024-12-05T11:58:37,982 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad38275ac3b44479adbaf585596bb639, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733399914979 2024-12-05T11:58:37,983 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:37,983 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/B is initiating minor compaction (all files) 2024-12-05T11:58:37,983 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/B in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:37,983 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/27262a7e4ed842b792fc649a34cb5620, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/558c560e57b44ed283283a0ee3c8acec, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/6831e4bd1ef447588d9fdcc5f0d1c524] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=36.7 K 2024-12-05T11:58:37,983 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7869087b720748d9a0eb0952ddc96779, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733399916915 2024-12-05T11:58:37,983 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 27262a7e4ed842b792fc649a34cb5620, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733399913673 2024-12-05T11:58:37,984 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 558c560e57b44ed283283a0ee3c8acec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733399914979 2024-12-05T11:58:37,984 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6831e4bd1ef447588d9fdcc5f0d1c524, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733399916915 2024-12-05T11:58:38,011 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#A#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:38,012 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/e06ed346475348eda39f07f2c8743c23 is 50, key is test_row_0/A:col10/1733399916915/Put/seqid=0 2024-12-05T11:58:38,024 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#B#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:38,025 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/e967ca4c57094cc8ad229e2bcb571877 is 50, key is test_row_0/B:col10/1733399916915/Put/seqid=0 2024-12-05T11:58:38,035 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T11:58:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T11:58:38,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T11:58:38,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:38,049 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-05T11:58:38,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742319_1495 (size=13051) 2024-12-05T11:58:38,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/05ec86c0ca504116977fad2368e9f05d is 50, key is test_row_0/A:col10/1733399917026/Put/seqid=0 2024-12-05T11:58:38,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742318_1494 (size=13051) 2024-12-05T11:58:38,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742320_1496 (size=12301) 2024-12-05T11:58:38,178 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/05ec86c0ca504116977fad2368e9f05d 2024-12-05T11:58:38,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/9abbce09c781432091927cdbc80f4ff6 is 50, key is test_row_0/B:col10/1733399917026/Put/seqid=0 2024-12-05T11:58:38,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:38,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742321_1497 (size=12301) 2024-12-05T11:58:38,247 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/9abbce09c781432091927cdbc80f4ff6 2024-12-05T11:58:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/71dacbf277824f7096e1b90c6360c953 is 50, key is test_row_0/C:col10/1733399917026/Put/seqid=0 2024-12-05T11:58:38,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399978285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:38,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399978289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:38,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742322_1498 (size=12301) 2024-12-05T11:58:38,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399978300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:38,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399978408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:38,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399978418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:38,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399978430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:38,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:38,494 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/e967ca4c57094cc8ad229e2bcb571877 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/e967ca4c57094cc8ad229e2bcb571877 2024-12-05T11:58:38,501 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/B of e6ea33c1931a2d534a8c12ecd9fa9d12 into e967ca4c57094cc8ad229e2bcb571877(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:38,501 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:38,501 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/B, priority=13, startTime=1733399917981; duration=0sec 2024-12-05T11:58:38,501 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:38,501 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:B 2024-12-05T11:58:38,501 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:38,505 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:38,505 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/C is initiating minor compaction (all files) 2024-12-05T11:58:38,505 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/C in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:38,505 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/437857e0cecf4ccc9c1bde30f6d7b557, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1fc15a7900d54725aaabc3990ee90765, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/4194344bab2847f9b13efd548ce95ff5] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=36.7 K 2024-12-05T11:58:38,505 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 437857e0cecf4ccc9c1bde30f6d7b557, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733399913673 2024-12-05T11:58:38,505 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fc15a7900d54725aaabc3990ee90765, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733399914979 2024-12-05T11:58:38,506 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4194344bab2847f9b13efd548ce95ff5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733399916915 2024-12-05T11:58:38,522 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#C#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:38,522 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/30f55169571f4366a9a4797bf2bfa705 is 50, key is test_row_0/C:col10/1733399916915/Put/seqid=0 2024-12-05T11:58:38,525 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/e06ed346475348eda39f07f2c8743c23 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/e06ed346475348eda39f07f2c8743c23 2024-12-05T11:58:38,530 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/A of e6ea33c1931a2d534a8c12ecd9fa9d12 into e06ed346475348eda39f07f2c8743c23(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:38,530 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:38,530 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/A, priority=13, startTime=1733399917980; duration=0sec 2024-12-05T11:58:38,530 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:38,530 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:A 2024-12-05T11:58:38,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742323_1499 (size=13051) 2024-12-05T11:58:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T11:58:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399978627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399978627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:38,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399978638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:38,713 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/71dacbf277824f7096e1b90c6360c953 2024-12-05T11:58:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/05ec86c0ca504116977fad2368e9f05d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/05ec86c0ca504116977fad2368e9f05d 2024-12-05T11:58:38,734 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/05ec86c0ca504116977fad2368e9f05d, entries=150, sequenceid=347, filesize=12.0 K 2024-12-05T11:58:38,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/9abbce09c781432091927cdbc80f4ff6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/9abbce09c781432091927cdbc80f4ff6 2024-12-05T11:58:38,746 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/9abbce09c781432091927cdbc80f4ff6, entries=150, sequenceid=347, filesize=12.0 K 2024-12-05T11:58:38,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/71dacbf277824f7096e1b90c6360c953 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/71dacbf277824f7096e1b90c6360c953 2024-12-05T11:58:38,756 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/71dacbf277824f7096e1b90c6360c953, entries=150, sequenceid=347, filesize=12.0 K 2024-12-05T11:58:38,757 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 707ms, sequenceid=347, compaction requested=false 2024-12-05T11:58:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-05T11:58:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-05T11:58:38,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-05T11:58:38,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3450 sec 2024-12-05T11:58:38,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3650 sec 2024-12-05T11:58:38,950 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/30f55169571f4366a9a4797bf2bfa705 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/30f55169571f4366a9a4797bf2bfa705 2024-12-05T11:58:38,951 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:38,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-05T11:58:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:38,959 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/C of e6ea33c1931a2d534a8c12ecd9fa9d12 into 30f55169571f4366a9a4797bf2bfa705(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:38,959 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:38,959 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/C, priority=13, startTime=1733399917981; duration=0sec 2024-12-05T11:58:38,959 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:38,959 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:C 2024-12-05T11:58:38,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/8bfa90784d674dd694fa2a3aae15d7c9 is 50, key is test_row_0/A:col10/1733399918286/Put/seqid=0 2024-12-05T11:58:39,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742324_1500 (size=12301) 2024-12-05T11:58:39,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/8bfa90784d674dd694fa2a3aae15d7c9 2024-12-05T11:58:39,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/472c7f51090144af94a1cceb94d61e81 is 50, key is test_row_0/B:col10/1733399918286/Put/seqid=0 2024-12-05T11:58:39,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742325_1501 (size=12301) 2024-12-05T11:58:39,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399979042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399979044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399979055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399979179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399979179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399979188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399979398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399979398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399979398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/472c7f51090144af94a1cceb94d61e81 2024-12-05T11:58:39,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/f3e5234c50b24ad683543bf465468f67 is 50, key is test_row_0/C:col10/1733399918286/Put/seqid=0 2024-12-05T11:58:39,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742326_1502 (size=12301) 2024-12-05T11:58:39,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/f3e5234c50b24ad683543bf465468f67 2024-12-05T11:58:39,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/8bfa90784d674dd694fa2a3aae15d7c9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/8bfa90784d674dd694fa2a3aae15d7c9 2024-12-05T11:58:39,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/8bfa90784d674dd694fa2a3aae15d7c9, entries=150, sequenceid=367, filesize=12.0 K 2024-12-05T11:58:39,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/472c7f51090144af94a1cceb94d61e81 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/472c7f51090144af94a1cceb94d61e81 2024-12-05T11:58:39,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/472c7f51090144af94a1cceb94d61e81, entries=150, sequenceid=367, filesize=12.0 K 2024-12-05T11:58:39,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/f3e5234c50b24ad683543bf465468f67 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/f3e5234c50b24ad683543bf465468f67 2024-12-05T11:58:39,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/f3e5234c50b24ad683543bf465468f67, entries=150, sequenceid=367, filesize=12.0 K 2024-12-05T11:58:39,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 593ms, sequenceid=367, compaction requested=true 2024-12-05T11:58:39,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:39,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:39,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:39,544 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:39,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:39,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:39,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:39,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T11:58:39,545 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:39,547 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:39,547 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:39,547 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/B is initiating minor compaction (all files) 2024-12-05T11:58:39,547 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/A is initiating minor compaction (all files) 2024-12-05T11:58:39,547 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/A in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:39,548 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/B in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:39,548 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/e967ca4c57094cc8ad229e2bcb571877, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/9abbce09c781432091927cdbc80f4ff6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/472c7f51090144af94a1cceb94d61e81] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=36.8 K 2024-12-05T11:58:39,548 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/e06ed346475348eda39f07f2c8743c23, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/05ec86c0ca504116977fad2368e9f05d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/8bfa90784d674dd694fa2a3aae15d7c9] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=36.8 K 2024-12-05T11:58:39,548 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting e967ca4c57094cc8ad229e2bcb571877, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733399916915 2024-12-05T11:58:39,548 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e06ed346475348eda39f07f2c8743c23, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733399916915 2024-12-05T11:58:39,548 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9abbce09c781432091927cdbc80f4ff6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733399917006 2024-12-05T11:58:39,548 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05ec86c0ca504116977fad2368e9f05d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733399917006 2024-12-05T11:58:39,548 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 472c7f51090144af94a1cceb94d61e81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733399918257 2024-12-05T11:58:39,549 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bfa90784d674dd694fa2a3aae15d7c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733399918257 2024-12-05T11:58:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T11:58:39,558 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:39,559 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-05T11:58:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T11:58:39,561 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:39,562 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:39,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:39,565 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:39,565 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/5bc6458752d14a9aaeeef15caf2d04b4 is 50, key is test_row_0/B:col10/1733399918286/Put/seqid=0 2024-12-05T11:58:39,578 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#A#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:39,578 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/59bec59aa2ee4514a295164d2f5dcd7a is 50, key is test_row_0/A:col10/1733399918286/Put/seqid=0 2024-12-05T11:58:39,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742327_1503 (size=13153) 2024-12-05T11:58:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T11:58:39,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742328_1504 (size=13153) 2024-12-05T11:58:39,685 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/59bec59aa2ee4514a295164d2f5dcd7a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/59bec59aa2ee4514a295164d2f5dcd7a 2024-12-05T11:58:39,690 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/A of e6ea33c1931a2d534a8c12ecd9fa9d12 into 59bec59aa2ee4514a295164d2f5dcd7a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:39,690 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:39,690 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/A, priority=13, startTime=1733399919544; duration=0sec 2024-12-05T11:58:39,690 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:39,690 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:A 2024-12-05T11:58:39,690 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:39,691 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:39,691 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/C is initiating minor compaction (all files) 2024-12-05T11:58:39,691 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/C in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:39,691 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/30f55169571f4366a9a4797bf2bfa705, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/71dacbf277824f7096e1b90c6360c953, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/f3e5234c50b24ad683543bf465468f67] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=36.8 K 2024-12-05T11:58:39,692 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30f55169571f4366a9a4797bf2bfa705, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733399916915 2024-12-05T11:58:39,692 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71dacbf277824f7096e1b90c6360c953, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733399917006 2024-12-05T11:58:39,692 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3e5234c50b24ad683543bf465468f67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733399918257 2024-12-05T11:58:39,703 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#C#compaction#432 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:39,704 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/cca7716a1dd946958578c741bf4522be is 50, key is test_row_0/C:col10/1733399918286/Put/seqid=0 2024-12-05T11:58:39,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-05T11:58:39,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:39,714 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-05T11:58:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:39,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:39,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742329_1505 (size=13153) 2024-12-05T11:58:39,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/7241098bf4b84f9c9bd82dc8496d44f7 is 50, key is test_row_0/A:col10/1733399919024/Put/seqid=0 2024-12-05T11:58:39,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742330_1506 (size=12301) 2024-12-05T11:58:39,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399979793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399979808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399979809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T11:58:39,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399979918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399979927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:39,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399979927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:39,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:39,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:40,047 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/5bc6458752d14a9aaeeef15caf2d04b4 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/5bc6458752d14a9aaeeef15caf2d04b4 2024-12-05T11:58:40,055 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/B of e6ea33c1931a2d534a8c12ecd9fa9d12 into 5bc6458752d14a9aaeeef15caf2d04b4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:40,055 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:40,055 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/B, priority=13, startTime=1733399919544; duration=0sec 2024-12-05T11:58:40,055 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:40,055 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:B 2024-12-05T11:58:40,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399980128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:40,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:40,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:40,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:40,142 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/cca7716a1dd946958578c741bf4522be as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/cca7716a1dd946958578c741bf4522be 2024-12-05T11:58:40,146 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/C of e6ea33c1931a2d534a8c12ecd9fa9d12 into cca7716a1dd946958578c741bf4522be(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:40,146 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:40,146 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/C, priority=13, startTime=1733399919545; duration=0sec 2024-12-05T11:58:40,146 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:40,146 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:C 2024-12-05T11:58:40,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399980148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:40,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399980149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:40,171 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/7241098bf4b84f9c9bd82dc8496d44f7 2024-12-05T11:58:40,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/ceedf95738054bf9a27155ff84e72955 is 50, key is test_row_0/B:col10/1733399919024/Put/seqid=0 2024-12-05T11:58:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T11:58:40,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742331_1507 (size=12301) 2024-12-05T11:58:40,216 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/ceedf95738054bf9a27155ff84e72955 2024-12-05T11:58:40,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/abd9060c5810478aab05f9a429b00d84 is 50, key is test_row_0/C:col10/1733399919024/Put/seqid=0 2024-12-05T11:58:40,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742332_1508 (size=12301) 2024-12-05T11:58:40,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399980449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:40,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399980468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:40,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399980468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:40,651 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/abd9060c5810478aab05f9a429b00d84 2024-12-05T11:58:40,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/7241098bf4b84f9c9bd82dc8496d44f7 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7241098bf4b84f9c9bd82dc8496d44f7 2024-12-05T11:58:40,662 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7241098bf4b84f9c9bd82dc8496d44f7, entries=150, sequenceid=387, filesize=12.0 K 2024-12-05T11:58:40,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/ceedf95738054bf9a27155ff84e72955 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/ceedf95738054bf9a27155ff84e72955 2024-12-05T11:58:40,676 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/ceedf95738054bf9a27155ff84e72955, entries=150, sequenceid=387, filesize=12.0 K 2024-12-05T11:58:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/abd9060c5810478aab05f9a429b00d84 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/abd9060c5810478aab05f9a429b00d84 2024-12-05T11:58:40,683 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/abd9060c5810478aab05f9a429b00d84, entries=150, sequenceid=387, filesize=12.0 K 2024-12-05T11:58:40,684 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 969ms, sequenceid=387, compaction requested=false 2024-12-05T11:58:40,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:40,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:40,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-05T11:58:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-05T11:58:40,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-05T11:58:40,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1230 sec 2024-12-05T11:58:40,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.1310 sec 2024-12-05T11:58:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T11:58:40,698 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:40,698 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-05T11:58:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T11:58:40,703 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:40,703 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:40,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T11:58:40,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-05T11:58:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:40,856 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-05T11:58:40,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/92a0495143164a3c9b1ecca8df846e50 is 50, key is test_row_0/A:col10/1733399919807/Put/seqid=0 2024-12-05T11:58:40,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742333_1509 (size=12301) 2024-12-05T11:58:40,892 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/92a0495143164a3c9b1ecca8df846e50 2024-12-05T11:58:40,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/fcbb98839d494ee19f8b9641a4d05980 is 50, key is test_row_0/B:col10/1733399919807/Put/seqid=0 2024-12-05T11:58:40,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742334_1510 (size=12301) 2024-12-05T11:58:40,922 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/fcbb98839d494ee19f8b9641a4d05980 2024-12-05T11:58:40,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/056e6e9865df42ccb13de06637c419a2 is 50, key is test_row_0/C:col10/1733399919807/Put/seqid=0 2024-12-05T11:58:40,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742335_1511 (size=12301) 2024-12-05T11:58:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:40,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T11:58:41,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399981025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399981043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399981044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49020 deadline: 1733399981090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,098 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in e6ea33c1931a2d534a8c12ecd9fa9d12 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399981147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399981168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399981168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T11:58:41,367 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/056e6e9865df42ccb13de06637c419a2 2024-12-05T11:58:41,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399981358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/92a0495143164a3c9b1ecca8df846e50 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/92a0495143164a3c9b1ecca8df846e50 2024-12-05T11:58:41,379 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/92a0495143164a3c9b1ecca8df846e50, entries=150, sequenceid=407, filesize=12.0 K 2024-12-05T11:58:41,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/fcbb98839d494ee19f8b9641a4d05980 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/fcbb98839d494ee19f8b9641a4d05980 2024-12-05T11:58:41,386 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/fcbb98839d494ee19f8b9641a4d05980, entries=150, sequenceid=407, filesize=12.0 K 2024-12-05T11:58:41,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/056e6e9865df42ccb13de06637c419a2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/056e6e9865df42ccb13de06637c419a2 2024-12-05T11:58:41,393 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/056e6e9865df42ccb13de06637c419a2, entries=150, sequenceid=407, filesize=12.0 K 2024-12-05T11:58:41,394 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 537ms, sequenceid=407, compaction requested=true 2024-12-05T11:58:41,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:41,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:41,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-05T11:58:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-05T11:58:41,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-05T11:58:41,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 701 msec 2024-12-05T11:58:41,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 693 msec 2024-12-05T11:58:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:41,412 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:41,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-05T11:58:41,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/80d5410db02445c3b1f5a70475ad647e is 50, key is test_row_0/A:col10/1733399921408/Put/seqid=0 2024-12-05T11:58:41,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742336_1512 (size=12301) 2024-12-05T11:58:41,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399981533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399981536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399981647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399981658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399981678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T11:58:41,838 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:41,839 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-05T11:58:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T11:58:41,841 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:41,842 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:41,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:41,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399981868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/80d5410db02445c3b1f5a70475ad647e 2024-12-05T11:58:41,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399981888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:41,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/d7c900c27751460dbb641c8c770bb878 is 50, key is test_row_0/B:col10/1733399921408/Put/seqid=0 2024-12-05T11:58:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T11:58:41,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742337_1513 (size=12301) 2024-12-05T11:58:41,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/d7c900c27751460dbb641c8c770bb878 2024-12-05T11:58:41,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/1499a2eba17d47f58267287389a90f2d is 50, key is test_row_0/C:col10/1733399921408/Put/seqid=0 2024-12-05T11:58:41,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-05T11:58:41,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:41,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:41,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:41,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:41,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:42,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742338_1514 (size=12301) 2024-12-05T11:58:42,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/1499a2eba17d47f58267287389a90f2d 2024-12-05T11:58:42,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/80d5410db02445c3b1f5a70475ad647e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/80d5410db02445c3b1f5a70475ad647e 2024-12-05T11:58:42,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/80d5410db02445c3b1f5a70475ad647e, entries=150, sequenceid=427, filesize=12.0 K 2024-12-05T11:58:42,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/d7c900c27751460dbb641c8c770bb878 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/d7c900c27751460dbb641c8c770bb878 2024-12-05T11:58:42,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/d7c900c27751460dbb641c8c770bb878, entries=150, sequenceid=427, filesize=12.0 K 2024-12-05T11:58:42,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/1499a2eba17d47f58267287389a90f2d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1499a2eba17d47f58267287389a90f2d 2024-12-05T11:58:42,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1499a2eba17d47f58267287389a90f2d, entries=150, sequenceid=427, filesize=12.0 K 2024-12-05T11:58:42,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 649ms, sequenceid=427, compaction requested=true 2024-12-05T11:58:42,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:42,061 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:42,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:42,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:42,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:42,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:42,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:42,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T11:58:42,062 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:42,065 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:42,065 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/A is initiating minor compaction (all files) 2024-12-05T11:58:42,065 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/A in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:42,065 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/59bec59aa2ee4514a295164d2f5dcd7a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7241098bf4b84f9c9bd82dc8496d44f7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/92a0495143164a3c9b1ecca8df846e50, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/80d5410db02445c3b1f5a70475ad647e] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=48.9 K 2024-12-05T11:58:42,067 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:42,067 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/B is initiating minor compaction (all files) 2024-12-05T11:58:42,067 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/B in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:42,067 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/5bc6458752d14a9aaeeef15caf2d04b4, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/ceedf95738054bf9a27155ff84e72955, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/fcbb98839d494ee19f8b9641a4d05980, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/d7c900c27751460dbb641c8c770bb878] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=48.9 K 2024-12-05T11:58:42,067 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59bec59aa2ee4514a295164d2f5dcd7a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733399918257 2024-12-05T11:58:42,067 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bc6458752d14a9aaeeef15caf2d04b4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733399918257 2024-12-05T11:58:42,067 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7241098bf4b84f9c9bd82dc8496d44f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733399919024 2024-12-05T11:58:42,068 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ceedf95738054bf9a27155ff84e72955, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733399919024 2024-12-05T11:58:42,068 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92a0495143164a3c9b1ecca8df846e50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733399919792 2024-12-05T11:58:42,068 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting fcbb98839d494ee19f8b9641a4d05980, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733399919792 2024-12-05T11:58:42,068 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80d5410db02445c3b1f5a70475ad647e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1733399921038 2024-12-05T11:58:42,068 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c900c27751460dbb641c8c770bb878, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1733399921038 2024-12-05T11:58:42,081 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#B#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:42,081 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/484d699798434ecb96cb83c9193c0b74 is 50, key is test_row_0/B:col10/1733399921408/Put/seqid=0 2024-12-05T11:58:42,101 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#A#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:42,101 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/b5ef643e377a413e89e5702b00d8fd50 is 50, key is test_row_0/A:col10/1733399921408/Put/seqid=0 2024-12-05T11:58:42,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-05T11:58:42,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:42,148 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-05T11:58:42,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742339_1515 (size=13289) 2024-12-05T11:58:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T11:58:42,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/7f56712f17a24caea51aae8127206db3 is 50, key is test_row_0/A:col10/1733399921530/Put/seqid=0 2024-12-05T11:58:42,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742340_1516 (size=13289) 2024-12-05T11:58:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. as already flushing 2024-12-05T11:58:42,200 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/b5ef643e377a413e89e5702b00d8fd50 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/b5ef643e377a413e89e5702b00d8fd50 2024-12-05T11:58:42,205 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/A of e6ea33c1931a2d534a8c12ecd9fa9d12 into b5ef643e377a413e89e5702b00d8fd50(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:42,205 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:42,205 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/A, priority=12, startTime=1733399922061; duration=0sec 2024-12-05T11:58:42,205 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:42,205 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:A 2024-12-05T11:58:42,205 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:58:42,206 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:58:42,206 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/C is initiating minor compaction (all files) 2024-12-05T11:58:42,206 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/C in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:42,206 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/cca7716a1dd946958578c741bf4522be, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/abd9060c5810478aab05f9a429b00d84, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/056e6e9865df42ccb13de06637c419a2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1499a2eba17d47f58267287389a90f2d] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=48.9 K 2024-12-05T11:58:42,207 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting cca7716a1dd946958578c741bf4522be, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733399918257 2024-12-05T11:58:42,207 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting abd9060c5810478aab05f9a429b00d84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733399919024 2024-12-05T11:58:42,207 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 056e6e9865df42ccb13de06637c419a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733399919792 2024-12-05T11:58:42,207 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1499a2eba17d47f58267287389a90f2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1733399921038 2024-12-05T11:58:42,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742341_1517 (size=12301) 2024-12-05T11:58:42,230 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/7f56712f17a24caea51aae8127206db3 2024-12-05T11:58:42,249 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#C#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:42,249 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/8d8fec912c70418ea2b32242d37c7fd4 is 50, key is test_row_0/C:col10/1733399921408/Put/seqid=0 2024-12-05T11:58:42,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/2fdbad4125814bfb897de3501e0ba339 is 50, key is test_row_0/B:col10/1733399921530/Put/seqid=0 2024-12-05T11:58:42,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399982276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:42,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399982278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:42,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399982279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:42,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:42,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:42,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742342_1518 (size=13289) 2024-12-05T11:58:42,305 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/8d8fec912c70418ea2b32242d37c7fd4 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/8d8fec912c70418ea2b32242d37c7fd4 2024-12-05T11:58:42,314 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/C of e6ea33c1931a2d534a8c12ecd9fa9d12 into 8d8fec912c70418ea2b32242d37c7fd4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:42,314 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:42,314 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/C, priority=12, startTime=1733399922062; duration=0sec 2024-12-05T11:58:42,314 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:42,314 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:C 2024-12-05T11:58:42,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742343_1519 (size=12301) 2024-12-05T11:58:42,328 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/2fdbad4125814bfb897de3501e0ba339 2024-12-05T11:58:42,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/c51b64e2c99546e39ad1c942a0f0f2ae is 50, key is test_row_0/C:col10/1733399921530/Put/seqid=0 2024-12-05T11:58:42,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742344_1520 (size=12301) 2024-12-05T11:58:42,387 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/c51b64e2c99546e39ad1c942a0f0f2ae 2024-12-05T11:58:42,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/7f56712f17a24caea51aae8127206db3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7f56712f17a24caea51aae8127206db3 2024-12-05T11:58:42,402 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7f56712f17a24caea51aae8127206db3, entries=150, sequenceid=443, filesize=12.0 K 2024-12-05T11:58:42,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1733399982398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:42,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49016 deadline: 1733399982398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:42,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49054 deadline: 1733399982398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 , the old value is region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:42,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/2fdbad4125814bfb897de3501e0ba339 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/2fdbad4125814bfb897de3501e0ba339 2024-12-05T11:58:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., hostname=80666a11a09f,40121,1733399769572, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:58:42,413 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/2fdbad4125814bfb897de3501e0ba339, entries=150, sequenceid=443, filesize=12.0 K 2024-12-05T11:58:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/c51b64e2c99546e39ad1c942a0f0f2ae as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c51b64e2c99546e39ad1c942a0f0f2ae 2024-12-05T11:58:42,421 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c51b64e2c99546e39ad1c942a0f0f2ae, entries=150, sequenceid=443, filesize=12.0 K 2024-12-05T11:58:42,422 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 274ms, sequenceid=443, compaction requested=false 2024-12-05T11:58:42,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:42,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:42,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-05T11:58:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-05T11:58:42,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-05T11:58:42,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 583 msec 2024-12-05T11:58:42,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 590 msec 2024-12-05T11:58:42,462 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-12-05T11:58:42,462 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:42,462 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,462 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:42,466 INFO [Thread-1939 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1939. 2024-12-05T11:58:42,466 DEBUG [Thread-1939 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:42,466 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,466 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,466 INFO [Thread-1941 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1941. 2024-12-05T11:58:42,467 DEBUG [Thread-1941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:42,467 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,467 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:42,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T11:58:42,468 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:42,468 INFO [Thread-1943 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1943. 2024-12-05T11:58:42,468 DEBUG [Thread-1943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:42,468 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,468 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:42,474 INFO [Thread-1947 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1947. 2024-12-05T11:58:42,474 DEBUG [Thread-1947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:42,474 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,474 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:42,561 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/484d699798434ecb96cb83c9193c0b74 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/484d699798434ecb96cb83c9193c0b74 2024-12-05T11:58:42,567 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/B of e6ea33c1931a2d534a8c12ecd9fa9d12 into 484d699798434ecb96cb83c9193c0b74(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:42,567 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:42,567 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/B, priority=12, startTime=1733399922062; duration=0sec 2024-12-05T11:58:42,567 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:42,567 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:B 2024-12-05T11:58:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:42,610 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:42,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T11:58:42,610 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-12-05T11:58:42,610 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-12-05T11:58:42,610 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:42,610 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:42,610 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,610 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,610 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,610 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:42,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:42,611 INFO [Thread-1928 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1928. 2024-12-05T11:58:42,611 DEBUG [Thread-1928 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:42,611 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,611 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:42,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:42,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/ee2646d90cda40929f125890e921976c is 50, key is test_row_0/A:col10/1733399922608/Put/seqid=0 2024-12-05T11:58:42,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742345_1521 (size=12301) 2024-12-05T11:58:43,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/ee2646d90cda40929f125890e921976c 2024-12-05T11:58:43,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/088db85d011c458a950c561a040e795c is 50, key is test_row_0/B:col10/1733399922608/Put/seqid=0 2024-12-05T11:58:43,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742346_1522 (size=12301) 2024-12-05T11:58:43,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/088db85d011c458a950c561a040e795c 2024-12-05T11:58:43,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/76c3487cb13449ea86cbafbd22cda762 is 50, key is test_row_0/C:col10/1733399922608/Put/seqid=0 2024-12-05T11:58:43,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742347_1523 (size=12301) 2024-12-05T11:58:43,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/76c3487cb13449ea86cbafbd22cda762 2024-12-05T11:58:43,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/ee2646d90cda40929f125890e921976c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ee2646d90cda40929f125890e921976c 2024-12-05T11:58:43,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ee2646d90cda40929f125890e921976c, entries=150, sequenceid=469, filesize=12.0 K 2024-12-05T11:58:43,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/088db85d011c458a950c561a040e795c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/088db85d011c458a950c561a040e795c 2024-12-05T11:58:43,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/088db85d011c458a950c561a040e795c, entries=150, sequenceid=469, filesize=12.0 K 2024-12-05T11:58:43,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/76c3487cb13449ea86cbafbd22cda762 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/76c3487cb13449ea86cbafbd22cda762 2024-12-05T11:58:43,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/76c3487cb13449ea86cbafbd22cda762, entries=150, sequenceid=469, filesize=12.0 K 2024-12-05T11:58:43,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 526ms, sequenceid=469, compaction requested=true 2024-12-05T11:58:43,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:43,137 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:43,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:43,138 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:43,138 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/A is initiating minor compaction (all files) 2024-12-05T11:58:43,138 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/A in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:43,138 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/b5ef643e377a413e89e5702b00d8fd50, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7f56712f17a24caea51aae8127206db3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ee2646d90cda40929f125890e921976c] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=37.0 K 2024-12-05T11:58:43,139 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5ef643e377a413e89e5702b00d8fd50, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1733399921038 2024-12-05T11:58:43,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:43,139 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:43,139 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f56712f17a24caea51aae8127206db3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733399921428 2024-12-05T11:58:43,139 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee2646d90cda40929f125890e921976c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733399922259 2024-12-05T11:58:43,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:43,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:43,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6ea33c1931a2d534a8c12ecd9fa9d12:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:43,140 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:43,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:43,140 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/B is initiating minor compaction (all files) 2024-12-05T11:58:43,140 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/B in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:43,140 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/484d699798434ecb96cb83c9193c0b74, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/2fdbad4125814bfb897de3501e0ba339, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/088db85d011c458a950c561a040e795c] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=37.0 K 2024-12-05T11:58:43,141 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 484d699798434ecb96cb83c9193c0b74, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1733399921038 2024-12-05T11:58:43,141 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fdbad4125814bfb897de3501e0ba339, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733399921428 2024-12-05T11:58:43,141 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 088db85d011c458a950c561a040e795c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733399922259 2024-12-05T11:58:43,156 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#A#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:43,157 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/ccd6f7297c864df1ba95e89fea7869d6 is 50, key is test_row_0/A:col10/1733399922608/Put/seqid=0 2024-12-05T11:58:43,172 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#B#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:43,173 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/d7f975620db3473d98d27dea9cf69be3 is 50, key is test_row_0/B:col10/1733399922608/Put/seqid=0 2024-12-05T11:58:43,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742348_1524 (size=13391) 2024-12-05T11:58:43,208 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/ccd6f7297c864df1ba95e89fea7869d6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ccd6f7297c864df1ba95e89fea7869d6 2024-12-05T11:58:43,217 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/A of e6ea33c1931a2d534a8c12ecd9fa9d12 into ccd6f7297c864df1ba95e89fea7869d6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:43,217 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:43,217 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/A, priority=13, startTime=1733399923137; duration=0sec 2024-12-05T11:58:43,217 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:43,217 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:A 2024-12-05T11:58:43,217 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:43,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742349_1525 (size=13391) 2024-12-05T11:58:43,219 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:43,220 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): e6ea33c1931a2d534a8c12ecd9fa9d12/C is initiating minor compaction (all files) 2024-12-05T11:58:43,220 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6ea33c1931a2d534a8c12ecd9fa9d12/C in TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:43,220 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/8d8fec912c70418ea2b32242d37c7fd4, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c51b64e2c99546e39ad1c942a0f0f2ae, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/76c3487cb13449ea86cbafbd22cda762] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp, totalSize=37.0 K 2024-12-05T11:58:43,220 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d8fec912c70418ea2b32242d37c7fd4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1733399921038 2024-12-05T11:58:43,220 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c51b64e2c99546e39ad1c942a0f0f2ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733399921428 2024-12-05T11:58:43,221 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76c3487cb13449ea86cbafbd22cda762, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733399922259 2024-12-05T11:58:43,230 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6ea33c1931a2d534a8c12ecd9fa9d12#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:43,231 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/dca8d6894e3e45e68d64015c23cb995f is 50, key is test_row_0/C:col10/1733399922608/Put/seqid=0 2024-12-05T11:58:43,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742350_1526 (size=13391) 2024-12-05T11:58:43,264 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/dca8d6894e3e45e68d64015c23cb995f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/dca8d6894e3e45e68d64015c23cb995f 2024-12-05T11:58:43,271 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/C of e6ea33c1931a2d534a8c12ecd9fa9d12 into dca8d6894e3e45e68d64015c23cb995f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:43,271 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:43,271 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/C, priority=13, startTime=1733399923140; duration=0sec 2024-12-05T11:58:43,271 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:43,271 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:C 2024-12-05T11:58:43,627 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/d7f975620db3473d98d27dea9cf69be3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/d7f975620db3473d98d27dea9cf69be3 2024-12-05T11:58:43,633 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6ea33c1931a2d534a8c12ecd9fa9d12/B of e6ea33c1931a2d534a8c12ecd9fa9d12 into d7f975620db3473d98d27dea9cf69be3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:43,633 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6ea33c1931a2d534a8c12ecd9fa9d12: 2024-12-05T11:58:43,633 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., storeName=e6ea33c1931a2d534a8c12ecd9fa9d12/B, priority=13, startTime=1733399923139; duration=0sec 2024-12-05T11:58:43,633 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:43,633 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6ea33c1931a2d534a8c12ecd9fa9d12:B 2024-12-05T11:58:45,454 INFO [Thread-1932 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1932. 2024-12-05T11:58:45,454 DEBUG [Thread-1932 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:45,454 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:45,454 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:45,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:51,132 INFO [Thread-1936 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1936. 2024-12-05T11:58:51,132 DEBUG [Thread-1936 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:58:51,132 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:51,132 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:51,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1394 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4182 rows 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1401 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4194 rows 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1410 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4230 rows 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1395 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4185 rows 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1394 2024-12-05T11:58:51,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4182 rows 2024-12-05T11:58:51,133 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:58:51,133 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:58:51,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:51,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:51,133 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:58:51,134 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T11:58:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-05T11:58:51,138 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399931138"}]},"ts":"1733399931138"} 2024-12-05T11:58:51,140 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T11:58:51,140 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T11:58:51,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:58:51,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6ea33c1931a2d534a8c12ecd9fa9d12, UNASSIGN}] 2024-12-05T11:58:51,144 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6ea33c1931a2d534a8c12ecd9fa9d12, UNASSIGN 2024-12-05T11:58:51,145 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e6ea33c1931a2d534a8c12ecd9fa9d12, regionState=CLOSING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:58:51,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6ea33c1931a2d534a8c12ecd9fa9d12, UNASSIGN because future has completed 2024-12-05T11:58:51,147 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:58:51,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:58:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-05T11:58:51,299 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:51,299 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:58:51,299 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing e6ea33c1931a2d534a8c12ecd9fa9d12, disabling compactions & flushes 2024-12-05T11:58:51,299 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:51,299 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:51,299 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. after waiting 0 ms 2024-12-05T11:58:51,299 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:51,299 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing e6ea33c1931a2d534a8c12ecd9fa9d12 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-12-05T11:58:51,303 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/fec19b1b8980473b95c43ea411563ba5 is 50, key is test_row_0/A:col10/1733399931130/Put/seqid=0 2024-12-05T11:58:51,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742351_1527 (size=9857) 2024-12-05T11:58:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-05T11:58:51,707 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/fec19b1b8980473b95c43ea411563ba5 2024-12-05T11:58:51,716 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/8d5871d08b8c405fb2d0727083c4aff2 is 50, key is test_row_0/B:col10/1733399931130/Put/seqid=0 2024-12-05T11:58:51,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742352_1528 (size=9857) 2024-12-05T11:58:51,764 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/8d5871d08b8c405fb2d0727083c4aff2 2024-12-05T11:58:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-05T11:58:51,773 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/962b077cf986492a82deab347db4d1ad is 50, key is test_row_0/C:col10/1733399931130/Put/seqid=0 2024-12-05T11:58:51,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742353_1529 (size=9857) 2024-12-05T11:58:51,796 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/962b077cf986492a82deab347db4d1ad 2024-12-05T11:58:51,800 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/A/fec19b1b8980473b95c43ea411563ba5 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/fec19b1b8980473b95c43ea411563ba5 2024-12-05T11:58:51,803 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/fec19b1b8980473b95c43ea411563ba5, entries=100, sequenceid=477, filesize=9.6 K 2024-12-05T11:58:51,804 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/B/8d5871d08b8c405fb2d0727083c4aff2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/8d5871d08b8c405fb2d0727083c4aff2 2024-12-05T11:58:51,807 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/8d5871d08b8c405fb2d0727083c4aff2, entries=100, sequenceid=477, filesize=9.6 K 2024-12-05T11:58:51,808 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/.tmp/C/962b077cf986492a82deab347db4d1ad as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/962b077cf986492a82deab347db4d1ad 2024-12-05T11:58:51,811 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/962b077cf986492a82deab347db4d1ad, entries=100, sequenceid=477, filesize=9.6 K 2024-12-05T11:58:51,812 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 513ms, sequenceid=477, compaction requested=false 2024-12-05T11:58:51,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/8f08950b7d9e4e34842a9b84f82819c2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/dcf7416e32b04286b3563b5590ff6620, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/9a0abc45583e4583814e01b9b44b418b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/3a6711ddfc8543b38e2ff231df674184, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/6766d2bfa7384faca56520228f6b24cf, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/a914d4d6ea9a435193aaeaccf691e0d5, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/70a474b8932a4d16a60826dc1fe78f9a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/1bc1c32fff854e34bff7ef61c9d4b450, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/bec112bfd1cd41e4ba1f07750868ad88, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/a9389f738ab24b44bc1d071985c64920, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/34a46a3a78c44c74a9b17f4b21ac2ac1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/bc0056d873454790a56a17632595be0e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7a159e4b47a5493dbe53e58e657abe68, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/eabb2b4b853d44cc9962fca32f947dd6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/0718b02d1b2749e489adbbda4c4f1e94, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ded38006a2b6449ea9f419fff373be93, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/1bf84ad90b4f43e18bba9a865226fa0c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/9587c73c9a8a4750b66fb66898fef2f3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/2435d33c1ed343f88aae916475bdf258, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/f985cb71c6e5464b835176aa3cfcede0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ad38275ac3b44479adbaf585596bb639, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/e06ed346475348eda39f07f2c8743c23, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7869087b720748d9a0eb0952ddc96779, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/05ec86c0ca504116977fad2368e9f05d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/59bec59aa2ee4514a295164d2f5dcd7a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/8bfa90784d674dd694fa2a3aae15d7c9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7241098bf4b84f9c9bd82dc8496d44f7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/92a0495143164a3c9b1ecca8df846e50, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/b5ef643e377a413e89e5702b00d8fd50, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/80d5410db02445c3b1f5a70475ad647e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7f56712f17a24caea51aae8127206db3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ee2646d90cda40929f125890e921976c] to archive 2024-12-05T11:58:51,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:58:51,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/8f08950b7d9e4e34842a9b84f82819c2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/8f08950b7d9e4e34842a9b84f82819c2 2024-12-05T11:58:51,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/dcf7416e32b04286b3563b5590ff6620 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/dcf7416e32b04286b3563b5590ff6620 2024-12-05T11:58:51,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/9a0abc45583e4583814e01b9b44b418b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/9a0abc45583e4583814e01b9b44b418b 2024-12-05T11:58:51,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/3a6711ddfc8543b38e2ff231df674184 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/3a6711ddfc8543b38e2ff231df674184 2024-12-05T11:58:51,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/6766d2bfa7384faca56520228f6b24cf to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/6766d2bfa7384faca56520228f6b24cf 2024-12-05T11:58:51,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/a914d4d6ea9a435193aaeaccf691e0d5 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/a914d4d6ea9a435193aaeaccf691e0d5 2024-12-05T11:58:51,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/70a474b8932a4d16a60826dc1fe78f9a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/70a474b8932a4d16a60826dc1fe78f9a 2024-12-05T11:58:51,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/1bc1c32fff854e34bff7ef61c9d4b450 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/1bc1c32fff854e34bff7ef61c9d4b450 2024-12-05T11:58:51,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/bec112bfd1cd41e4ba1f07750868ad88 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/bec112bfd1cd41e4ba1f07750868ad88 2024-12-05T11:58:51,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/a9389f738ab24b44bc1d071985c64920 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/a9389f738ab24b44bc1d071985c64920 2024-12-05T11:58:51,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/34a46a3a78c44c74a9b17f4b21ac2ac1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/34a46a3a78c44c74a9b17f4b21ac2ac1 2024-12-05T11:58:51,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/bc0056d873454790a56a17632595be0e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/bc0056d873454790a56a17632595be0e 2024-12-05T11:58:51,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7a159e4b47a5493dbe53e58e657abe68 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7a159e4b47a5493dbe53e58e657abe68 2024-12-05T11:58:51,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/eabb2b4b853d44cc9962fca32f947dd6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/eabb2b4b853d44cc9962fca32f947dd6 2024-12-05T11:58:51,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/0718b02d1b2749e489adbbda4c4f1e94 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/0718b02d1b2749e489adbbda4c4f1e94 2024-12-05T11:58:51,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ded38006a2b6449ea9f419fff373be93 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ded38006a2b6449ea9f419fff373be93 2024-12-05T11:58:51,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/1bf84ad90b4f43e18bba9a865226fa0c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/1bf84ad90b4f43e18bba9a865226fa0c 2024-12-05T11:58:51,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/9587c73c9a8a4750b66fb66898fef2f3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/9587c73c9a8a4750b66fb66898fef2f3 2024-12-05T11:58:51,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/2435d33c1ed343f88aae916475bdf258 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/2435d33c1ed343f88aae916475bdf258 2024-12-05T11:58:51,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/f985cb71c6e5464b835176aa3cfcede0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/f985cb71c6e5464b835176aa3cfcede0 2024-12-05T11:58:51,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ad38275ac3b44479adbaf585596bb639 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ad38275ac3b44479adbaf585596bb639 2024-12-05T11:58:51,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/e06ed346475348eda39f07f2c8743c23 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/e06ed346475348eda39f07f2c8743c23 2024-12-05T11:58:51,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7869087b720748d9a0eb0952ddc96779 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7869087b720748d9a0eb0952ddc96779 2024-12-05T11:58:51,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/05ec86c0ca504116977fad2368e9f05d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/05ec86c0ca504116977fad2368e9f05d 2024-12-05T11:58:51,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/59bec59aa2ee4514a295164d2f5dcd7a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/59bec59aa2ee4514a295164d2f5dcd7a 2024-12-05T11:58:51,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/8bfa90784d674dd694fa2a3aae15d7c9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/8bfa90784d674dd694fa2a3aae15d7c9 2024-12-05T11:58:51,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7241098bf4b84f9c9bd82dc8496d44f7 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7241098bf4b84f9c9bd82dc8496d44f7 2024-12-05T11:58:51,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/92a0495143164a3c9b1ecca8df846e50 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/92a0495143164a3c9b1ecca8df846e50 2024-12-05T11:58:51,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/b5ef643e377a413e89e5702b00d8fd50 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/b5ef643e377a413e89e5702b00d8fd50 2024-12-05T11:58:51,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/80d5410db02445c3b1f5a70475ad647e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/80d5410db02445c3b1f5a70475ad647e 2024-12-05T11:58:51,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7f56712f17a24caea51aae8127206db3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/7f56712f17a24caea51aae8127206db3 2024-12-05T11:58:51,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ee2646d90cda40929f125890e921976c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ee2646d90cda40929f125890e921976c 2024-12-05T11:58:51,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4fa2c613d50b48a49695e9835996262a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/147defc5dde6416c8624ba80c2313f5b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/5dbcb75ca9af49f087e90dd0f02103ac, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/02d7dc721607467c9367c9cb50ad4fff, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/b21203dd86514ac0b81c34b9e4580944, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4e5826ea20f34bbbaa09e6ab7031988f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/9afc208378c94274854a2bbc157df139, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/241db7e939264d89978f382d52989f0c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/2cd77760f35e4c3995bf08077346f9b8, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/ec28d13c3c5d45e29356e6bad4aff987, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/07c7808eb4534e6c9a8bf5f6405b5c3c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/fae163aca91244748a73d15f1edb085f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/e66cb2a317074f84a12d73f94da0fbf0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/84d7f62d4cf04fbb9566f9afbb6b4b30, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/aced7d0ff7684aefa7579c35d7c4b5dd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/cd4e9be06b09465da199cf12b5bb73dd, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/b52d0a9ff58d4ab0a79fc6909ab6fcca, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/6853978e232b4e288bb0537e790c81a5, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/27262a7e4ed842b792fc649a34cb5620, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4c31a54d3411404c8ce57f3dc33cbed8, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/558c560e57b44ed283283a0ee3c8acec, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/e967ca4c57094cc8ad229e2bcb571877, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/6831e4bd1ef447588d9fdcc5f0d1c524, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/9abbce09c781432091927cdbc80f4ff6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/5bc6458752d14a9aaeeef15caf2d04b4, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/472c7f51090144af94a1cceb94d61e81, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/ceedf95738054bf9a27155ff84e72955, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/fcbb98839d494ee19f8b9641a4d05980, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/484d699798434ecb96cb83c9193c0b74, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/d7c900c27751460dbb641c8c770bb878, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/2fdbad4125814bfb897de3501e0ba339, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/088db85d011c458a950c561a040e795c] to archive 2024-12-05T11:58:51,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:58:51,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4fa2c613d50b48a49695e9835996262a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4fa2c613d50b48a49695e9835996262a 2024-12-05T11:58:51,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/147defc5dde6416c8624ba80c2313f5b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/147defc5dde6416c8624ba80c2313f5b 2024-12-05T11:58:51,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/5dbcb75ca9af49f087e90dd0f02103ac to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/5dbcb75ca9af49f087e90dd0f02103ac 2024-12-05T11:58:51,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/02d7dc721607467c9367c9cb50ad4fff to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/02d7dc721607467c9367c9cb50ad4fff 2024-12-05T11:58:51,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/b21203dd86514ac0b81c34b9e4580944 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/b21203dd86514ac0b81c34b9e4580944 2024-12-05T11:58:51,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4e5826ea20f34bbbaa09e6ab7031988f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4e5826ea20f34bbbaa09e6ab7031988f 2024-12-05T11:58:51,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/9afc208378c94274854a2bbc157df139 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/9afc208378c94274854a2bbc157df139 2024-12-05T11:58:51,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/241db7e939264d89978f382d52989f0c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/241db7e939264d89978f382d52989f0c 2024-12-05T11:58:51,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/2cd77760f35e4c3995bf08077346f9b8 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/2cd77760f35e4c3995bf08077346f9b8 2024-12-05T11:58:51,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/ec28d13c3c5d45e29356e6bad4aff987 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/ec28d13c3c5d45e29356e6bad4aff987 2024-12-05T11:58:51,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/07c7808eb4534e6c9a8bf5f6405b5c3c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/07c7808eb4534e6c9a8bf5f6405b5c3c 2024-12-05T11:58:51,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/fae163aca91244748a73d15f1edb085f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/fae163aca91244748a73d15f1edb085f 2024-12-05T11:58:51,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/e66cb2a317074f84a12d73f94da0fbf0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/e66cb2a317074f84a12d73f94da0fbf0 2024-12-05T11:58:51,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/84d7f62d4cf04fbb9566f9afbb6b4b30 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/84d7f62d4cf04fbb9566f9afbb6b4b30 2024-12-05T11:58:51,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/aced7d0ff7684aefa7579c35d7c4b5dd to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/aced7d0ff7684aefa7579c35d7c4b5dd 2024-12-05T11:58:51,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/cd4e9be06b09465da199cf12b5bb73dd to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/cd4e9be06b09465da199cf12b5bb73dd 2024-12-05T11:58:51,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/b52d0a9ff58d4ab0a79fc6909ab6fcca to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/b52d0a9ff58d4ab0a79fc6909ab6fcca 2024-12-05T11:58:51,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/6853978e232b4e288bb0537e790c81a5 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/6853978e232b4e288bb0537e790c81a5 2024-12-05T11:58:51,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/27262a7e4ed842b792fc649a34cb5620 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/27262a7e4ed842b792fc649a34cb5620 2024-12-05T11:58:51,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4c31a54d3411404c8ce57f3dc33cbed8 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/4c31a54d3411404c8ce57f3dc33cbed8 2024-12-05T11:58:51,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/558c560e57b44ed283283a0ee3c8acec to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/558c560e57b44ed283283a0ee3c8acec 2024-12-05T11:58:51,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/e967ca4c57094cc8ad229e2bcb571877 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/e967ca4c57094cc8ad229e2bcb571877 2024-12-05T11:58:51,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/6831e4bd1ef447588d9fdcc5f0d1c524 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/6831e4bd1ef447588d9fdcc5f0d1c524 2024-12-05T11:58:51,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/9abbce09c781432091927cdbc80f4ff6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/9abbce09c781432091927cdbc80f4ff6 2024-12-05T11:58:51,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/5bc6458752d14a9aaeeef15caf2d04b4 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/5bc6458752d14a9aaeeef15caf2d04b4 2024-12-05T11:58:51,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/472c7f51090144af94a1cceb94d61e81 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/472c7f51090144af94a1cceb94d61e81 2024-12-05T11:58:51,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/ceedf95738054bf9a27155ff84e72955 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/ceedf95738054bf9a27155ff84e72955 2024-12-05T11:58:51,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/fcbb98839d494ee19f8b9641a4d05980 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/fcbb98839d494ee19f8b9641a4d05980 2024-12-05T11:58:51,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/484d699798434ecb96cb83c9193c0b74 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/484d699798434ecb96cb83c9193c0b74 2024-12-05T11:58:51,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/d7c900c27751460dbb641c8c770bb878 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/d7c900c27751460dbb641c8c770bb878 2024-12-05T11:58:51,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/2fdbad4125814bfb897de3501e0ba339 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/2fdbad4125814bfb897de3501e0ba339 2024-12-05T11:58:51,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/088db85d011c458a950c561a040e795c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/088db85d011c458a950c561a040e795c 2024-12-05T11:58:51,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c841a4102438483a8de9ece2bac24ac0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/f84db19094ce4d3e84f0ccc256e9a916, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/2858f591a08f4fc7968fb1916b466334, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/4d04e0866a6548fd935c00c3e6534e75, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/0e5ef276cdf744d6b37afb295255a68a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/299a76c94bbd40b3afd80c11fd42acb7, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/011cf4e372934fb0b12625e5612ab44f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c7db356762094c7baf64a76719091f7f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/7fa210b484554d6084c14cc8d5668534, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/18cd4ecbc24044ce848de08072532a7f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/47522f00999a49c0802ff77513289e54, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/6da0178c59f143dc9ea8e118d7d6dbf8, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/d94f418edd1d4239bd11c442693650d2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/3189a2423203407394311c50e9263b09, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/242749f685944504926af688c84c2bc3, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/cbd63a6c82ac4146986656bd65186194, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/b6fa15bd5d5e458493a501d370c89768, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1e3f1c300fce421a8afd903790bd584b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/437857e0cecf4ccc9c1bde30f6d7b557, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c0d8bae7277d43ff9a707903b425bb1d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1fc15a7900d54725aaabc3990ee90765, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/30f55169571f4366a9a4797bf2bfa705, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/4194344bab2847f9b13efd548ce95ff5, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/71dacbf277824f7096e1b90c6360c953, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/cca7716a1dd946958578c741bf4522be, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/f3e5234c50b24ad683543bf465468f67, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/abd9060c5810478aab05f9a429b00d84, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/056e6e9865df42ccb13de06637c419a2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/8d8fec912c70418ea2b32242d37c7fd4, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1499a2eba17d47f58267287389a90f2d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c51b64e2c99546e39ad1c942a0f0f2ae, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/76c3487cb13449ea86cbafbd22cda762] to archive 2024-12-05T11:58:51,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:58:51,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c841a4102438483a8de9ece2bac24ac0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c841a4102438483a8de9ece2bac24ac0 2024-12-05T11:58:51,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/f84db19094ce4d3e84f0ccc256e9a916 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/f84db19094ce4d3e84f0ccc256e9a916 2024-12-05T11:58:51,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/2858f591a08f4fc7968fb1916b466334 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/2858f591a08f4fc7968fb1916b466334 2024-12-05T11:58:51,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/4d04e0866a6548fd935c00c3e6534e75 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/4d04e0866a6548fd935c00c3e6534e75 2024-12-05T11:58:51,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/0e5ef276cdf744d6b37afb295255a68a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/0e5ef276cdf744d6b37afb295255a68a 2024-12-05T11:58:51,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/299a76c94bbd40b3afd80c11fd42acb7 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/299a76c94bbd40b3afd80c11fd42acb7 2024-12-05T11:58:51,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/011cf4e372934fb0b12625e5612ab44f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/011cf4e372934fb0b12625e5612ab44f 2024-12-05T11:58:51,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c7db356762094c7baf64a76719091f7f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c7db356762094c7baf64a76719091f7f 2024-12-05T11:58:51,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/7fa210b484554d6084c14cc8d5668534 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/7fa210b484554d6084c14cc8d5668534 2024-12-05T11:58:51,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/18cd4ecbc24044ce848de08072532a7f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/18cd4ecbc24044ce848de08072532a7f 2024-12-05T11:58:51,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/47522f00999a49c0802ff77513289e54 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/47522f00999a49c0802ff77513289e54 2024-12-05T11:58:51,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/6da0178c59f143dc9ea8e118d7d6dbf8 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/6da0178c59f143dc9ea8e118d7d6dbf8 2024-12-05T11:58:51,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/d94f418edd1d4239bd11c442693650d2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/d94f418edd1d4239bd11c442693650d2 2024-12-05T11:58:51,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/3189a2423203407394311c50e9263b09 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/3189a2423203407394311c50e9263b09 2024-12-05T11:58:51,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/242749f685944504926af688c84c2bc3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/242749f685944504926af688c84c2bc3 2024-12-05T11:58:51,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/cbd63a6c82ac4146986656bd65186194 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/cbd63a6c82ac4146986656bd65186194 2024-12-05T11:58:51,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/b6fa15bd5d5e458493a501d370c89768 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/b6fa15bd5d5e458493a501d370c89768 2024-12-05T11:58:51,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1e3f1c300fce421a8afd903790bd584b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1e3f1c300fce421a8afd903790bd584b 2024-12-05T11:58:51,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/437857e0cecf4ccc9c1bde30f6d7b557 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/437857e0cecf4ccc9c1bde30f6d7b557 2024-12-05T11:58:51,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c0d8bae7277d43ff9a707903b425bb1d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c0d8bae7277d43ff9a707903b425bb1d 2024-12-05T11:58:51,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1fc15a7900d54725aaabc3990ee90765 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1fc15a7900d54725aaabc3990ee90765 2024-12-05T11:58:51,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/30f55169571f4366a9a4797bf2bfa705 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/30f55169571f4366a9a4797bf2bfa705 2024-12-05T11:58:51,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/4194344bab2847f9b13efd548ce95ff5 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/4194344bab2847f9b13efd548ce95ff5 2024-12-05T11:58:51,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/71dacbf277824f7096e1b90c6360c953 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/71dacbf277824f7096e1b90c6360c953 2024-12-05T11:58:51,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/cca7716a1dd946958578c741bf4522be to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/cca7716a1dd946958578c741bf4522be 2024-12-05T11:58:51,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/f3e5234c50b24ad683543bf465468f67 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/f3e5234c50b24ad683543bf465468f67 2024-12-05T11:58:51,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/abd9060c5810478aab05f9a429b00d84 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/abd9060c5810478aab05f9a429b00d84 2024-12-05T11:58:51,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/056e6e9865df42ccb13de06637c419a2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/056e6e9865df42ccb13de06637c419a2 2024-12-05T11:58:51,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/8d8fec912c70418ea2b32242d37c7fd4 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/8d8fec912c70418ea2b32242d37c7fd4 2024-12-05T11:58:51,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1499a2eba17d47f58267287389a90f2d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/1499a2eba17d47f58267287389a90f2d 2024-12-05T11:58:51,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c51b64e2c99546e39ad1c942a0f0f2ae to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/c51b64e2c99546e39ad1c942a0f0f2ae 2024-12-05T11:58:51,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/76c3487cb13449ea86cbafbd22cda762 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/76c3487cb13449ea86cbafbd22cda762 2024-12-05T11:58:51,905 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/recovered.edits/480.seqid, newMaxSeqId=480, maxSeqId=1 2024-12-05T11:58:51,906 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. 2024-12-05T11:58:51,906 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for e6ea33c1931a2d534a8c12ecd9fa9d12: Waiting for close lock at 1733399931299Running coprocessor pre-close hooks at 1733399931299Disabling compacts and flushes for region at 1733399931299Disabling writes for close at 1733399931299Obtaining lock to block concurrent updates at 1733399931300 (+1 ms)Preparing flush snapshotting stores in e6ea33c1931a2d534a8c12ecd9fa9d12 at 1733399931300Finished memstore snapshotting TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1733399931300Flushing stores of TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12. at 1733399931300Flushing e6ea33c1931a2d534a8c12ecd9fa9d12/A: creating writer at 1733399931300Flushing e6ea33c1931a2d534a8c12ecd9fa9d12/A: appending metadata at 1733399931302 (+2 ms)Flushing e6ea33c1931a2d534a8c12ecd9fa9d12/A: closing flushed file at 1733399931302Flushing e6ea33c1931a2d534a8c12ecd9fa9d12/B: creating writer at 1733399931712 (+410 ms)Flushing e6ea33c1931a2d534a8c12ecd9fa9d12/B: appending metadata at 1733399931716 (+4 ms)Flushing e6ea33c1931a2d534a8c12ecd9fa9d12/B: closing flushed file at 1733399931716Flushing e6ea33c1931a2d534a8c12ecd9fa9d12/C: creating writer at 1733399931768 (+52 ms)Flushing e6ea33c1931a2d534a8c12ecd9fa9d12/C: appending metadata at 1733399931772 (+4 ms)Flushing e6ea33c1931a2d534a8c12ecd9fa9d12/C: closing flushed file at 1733399931772Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@408df43e: reopening flushed file at 1733399931799 (+27 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@457f2906: reopening flushed file at 1733399931803 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54c29acd: reopening flushed file at 1733399931807 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for e6ea33c1931a2d534a8c12ecd9fa9d12 in 513ms, sequenceid=477, compaction requested=false at 1733399931812 (+5 ms)Writing region close event to WAL at 1733399931902 (+90 ms)Running coprocessor post-close hooks at 1733399931906 (+4 ms)Closed at 1733399931906 2024-12-05T11:58:51,908 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:51,908 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e6ea33c1931a2d534a8c12ecd9fa9d12, regionState=CLOSED 2024-12-05T11:58:51,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:58:51,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-05T11:58:51,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure e6ea33c1931a2d534a8c12ecd9fa9d12, server=80666a11a09f,40121,1733399769572 in 763 msec 2024-12-05T11:58:51,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-05T11:58:51,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6ea33c1931a2d534a8c12ecd9fa9d12, UNASSIGN in 769 msec 2024-12-05T11:58:51,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-05T11:58:51,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 772 msec 2024-12-05T11:58:51,916 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399931916"}]},"ts":"1733399931916"} 2024-12-05T11:58:51,917 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T11:58:51,917 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T11:58:51,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 784 msec 2024-12-05T11:58:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-05T11:58:52,278 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:52,279 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T11:58:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:52,280 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-05T11:58:52,281 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:52,284 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:52,285 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/recovered.edits] 2024-12-05T11:58:52,292 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ccd6f7297c864df1ba95e89fea7869d6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/ccd6f7297c864df1ba95e89fea7869d6 2024-12-05T11:58:52,294 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/fec19b1b8980473b95c43ea411563ba5 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/A/fec19b1b8980473b95c43ea411563ba5 2024-12-05T11:58:52,295 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/8d5871d08b8c405fb2d0727083c4aff2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/8d5871d08b8c405fb2d0727083c4aff2 2024-12-05T11:58:52,296 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/d7f975620db3473d98d27dea9cf69be3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/B/d7f975620db3473d98d27dea9cf69be3 2024-12-05T11:58:52,303 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/962b077cf986492a82deab347db4d1ad to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/962b077cf986492a82deab347db4d1ad 2024-12-05T11:58:52,304 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/dca8d6894e3e45e68d64015c23cb995f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/C/dca8d6894e3e45e68d64015c23cb995f 2024-12-05T11:58:52,306 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/recovered.edits/480.seqid to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12/recovered.edits/480.seqid 2024-12-05T11:58:52,307 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/e6ea33c1931a2d534a8c12ecd9fa9d12 2024-12-05T11:58:52,307 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T11:58:52,310 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:52,312 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T11:58:52,313 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T11:58:52,314 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:52,314 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T11:58:52,314 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733399932314"}]},"ts":"9223372036854775807"} 2024-12-05T11:58:52,316 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T11:58:52,316 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e6ea33c1931a2d534a8c12ecd9fa9d12, NAME => 'TestAcidGuarantees,,1733399900225.e6ea33c1931a2d534a8c12ecd9fa9d12.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T11:58:52,317 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T11:58:52,317 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733399932317"}]},"ts":"9223372036854775807"} 2024-12-05T11:58:52,319 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T11:58:52,320 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:52,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-12-05T11:58:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-05T11:58:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T11:58:52,391 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:52,398 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239 (was 241), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=562 (was 344) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6829 (was 7270) 2024-12-05T11:58:52,407 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=6829 2024-12-05T11:58:52,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:58:52,409 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:58:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:52,411 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T11:58:52,411 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:52,411 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-12-05T11:58:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T11:58:52,412 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T11:58:52,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742354_1530 (size=959) 2024-12-05T11:58:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T11:58:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T11:58:52,818 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c 2024-12-05T11:58:52,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742355_1531 (size=53) 2024-12-05T11:58:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T11:58:53,223 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:58:53,223 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 990ce2fe1eb6f27987cd2f0b966a4627, disabling compactions & flushes 2024-12-05T11:58:53,223 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:53,223 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:53,223 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. after waiting 0 ms 2024-12-05T11:58:53,223 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:53,223 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:53,223 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 990ce2fe1eb6f27987cd2f0b966a4627: Waiting for close lock at 1733399933223Disabling compacts and flushes for region at 1733399933223Disabling writes for close at 1733399933223Writing region close event to WAL at 1733399933223Closed at 1733399933223 2024-12-05T11:58:53,224 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T11:58:53,225 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733399933224"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733399933224"}]},"ts":"1733399933224"} 2024-12-05T11:58:53,226 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T11:58:53,227 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T11:58:53,227 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399933227"}]},"ts":"1733399933227"} 2024-12-05T11:58:53,228 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T11:58:53,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, ASSIGN}] 2024-12-05T11:58:53,229 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, ASSIGN 2024-12-05T11:58:53,230 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, ASSIGN; state=OFFLINE, location=80666a11a09f,40121,1733399769572; forceNewPlan=false, retain=false 2024-12-05T11:58:53,380 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=990ce2fe1eb6f27987cd2f0b966a4627, regionState=OPENING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:58:53,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, ASSIGN because future has completed 2024-12-05T11:58:53,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:58:53,538 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:53,539 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:58:53,539 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,539 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:58:53,539 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,539 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,541 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,542 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 990ce2fe1eb6f27987cd2f0b966a4627 columnFamilyName A 2024-12-05T11:58:53,542 DEBUG [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:53,543 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] regionserver.HStore(327): Store=990ce2fe1eb6f27987cd2f0b966a4627/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:58:53,543 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,544 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 990ce2fe1eb6f27987cd2f0b966a4627 columnFamilyName B 2024-12-05T11:58:53,544 DEBUG [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:53,544 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] regionserver.HStore(327): Store=990ce2fe1eb6f27987cd2f0b966a4627/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:58:53,544 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,545 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 990ce2fe1eb6f27987cd2f0b966a4627 columnFamilyName C 2024-12-05T11:58:53,545 DEBUG [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:53,545 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] regionserver.HStore(327): Store=990ce2fe1eb6f27987cd2f0b966a4627/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:58:53,545 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,546 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,546 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,547 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,547 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,547 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:58:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T11:58:53,548 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,549 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:58:53,550 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 990ce2fe1eb6f27987cd2f0b966a4627; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74568908, jitterRate=0.11116331815719604}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:58:53,550 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:53,550 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 990ce2fe1eb6f27987cd2f0b966a4627: Running coprocessor pre-open hook at 1733399933539Writing region info on filesystem at 1733399933539Initializing all the Stores at 1733399933540 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399933540Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399933540Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399933540Cleaning up temporary data from old regions at 1733399933547 (+7 ms)Running coprocessor post-open hooks at 1733399933550 (+3 ms)Region opened successfully at 1733399933550 2024-12-05T11:58:53,551 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., pid=146, masterSystemTime=1733399933534 2024-12-05T11:58:53,552 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:53,552 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:53,553 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=990ce2fe1eb6f27987cd2f0b966a4627, regionState=OPEN, openSeqNum=2, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:58:53,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:58:53,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-05T11:58:53,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 in 172 msec 2024-12-05T11:58:53,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-05T11:58:53,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, ASSIGN in 328 msec 2024-12-05T11:58:53,558 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T11:58:53,558 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399933558"}]},"ts":"1733399933558"} 2024-12-05T11:58:53,560 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T11:58:53,560 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T11:58:53,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-12-05T11:58:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T11:58:54,569 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:54,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348042de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:54,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:54,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:54,575 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:54,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@530227f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:54,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:54,578 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e6bd52b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:54,579 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:54,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:54,581 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:58:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 80666a11a09f,33741,1733399768894 2024-12-05T11:58:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@90cacb3 2024-12-05T11:58:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:58:54,582 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:58:54,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:58:54,583 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:58:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T11:58:54,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742356_1532 (size=995) 2024-12-05T11:58:54,997 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-05T11:58:54,997 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-05T11:58:55,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:58:55,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, REOPEN/MOVE}] 2024-12-05T11:58:55,008 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, REOPEN/MOVE 2024-12-05T11:58:55,008 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=990ce2fe1eb6f27987cd2f0b966a4627, regionState=CLOSING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, REOPEN/MOVE because future has completed 2024-12-05T11:58:55,010 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:58:55,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:58:55,163 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,164 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:58:55,164 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 990ce2fe1eb6f27987cd2f0b966a4627, disabling compactions & flushes 2024-12-05T11:58:55,164 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:55,164 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:55,164 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. after waiting 0 ms 2024-12-05T11:58:55,164 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:55,174 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T11:58:55,175 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:55,175 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 990ce2fe1eb6f27987cd2f0b966a4627: Waiting for close lock at 1733399935164Running coprocessor pre-close hooks at 1733399935164Disabling compacts and flushes for region at 1733399935164Disabling writes for close at 1733399935164Writing region close event to WAL at 1733399935165 (+1 ms)Running coprocessor post-close hooks at 1733399935175 (+10 ms)Closed at 1733399935175 2024-12-05T11:58:55,175 WARN [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 990ce2fe1eb6f27987cd2f0b966a4627 to self. 2024-12-05T11:58:55,178 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,179 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=990ce2fe1eb6f27987cd2f0b966a4627, regionState=CLOSED 2024-12-05T11:58:55,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:58:55,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-05T11:58:55,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 in 173 msec 2024-12-05T11:58:55,187 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, REOPEN/MOVE; state=CLOSED, location=80666a11a09f,40121,1733399769572; forceNewPlan=false, retain=true 2024-12-05T11:58:55,338 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=990ce2fe1eb6f27987cd2f0b966a4627, regionState=OPENING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, REOPEN/MOVE because future has completed 2024-12-05T11:58:55,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:58:55,499 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:55,499 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:58:55,500 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,500 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:58:55,500 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,500 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,501 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,502 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 990ce2fe1eb6f27987cd2f0b966a4627 columnFamilyName A 2024-12-05T11:58:55,504 DEBUG [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:55,504 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] regionserver.HStore(327): Store=990ce2fe1eb6f27987cd2f0b966a4627/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:58:55,505 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,506 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 990ce2fe1eb6f27987cd2f0b966a4627 columnFamilyName B 2024-12-05T11:58:55,506 DEBUG [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:55,506 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] regionserver.HStore(327): Store=990ce2fe1eb6f27987cd2f0b966a4627/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:58:55,506 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,507 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 990ce2fe1eb6f27987cd2f0b966a4627 columnFamilyName C 2024-12-05T11:58:55,508 DEBUG [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:55,508 INFO [StoreOpener-990ce2fe1eb6f27987cd2f0b966a4627-1 {}] regionserver.HStore(327): Store=990ce2fe1eb6f27987cd2f0b966a4627/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:58:55,508 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,509 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,511 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,513 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,513 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,514 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:58:55,515 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,517 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 990ce2fe1eb6f27987cd2f0b966a4627; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70365209, jitterRate=0.048523321747779846}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:58:55,517 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,518 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 990ce2fe1eb6f27987cd2f0b966a4627: Running coprocessor pre-open hook at 1733399935500Writing region info on filesystem at 1733399935500Initializing all the Stores at 1733399935501 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399935501Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399935501Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733399935501Cleaning up temporary data from old regions at 1733399935513 (+12 ms)Running coprocessor post-open hooks at 1733399935517 (+4 ms)Region opened successfully at 1733399935517 2024-12-05T11:58:55,518 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., pid=151, masterSystemTime=1733399935496 2024-12-05T11:58:55,521 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:55,521 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:55,522 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=990ce2fe1eb6f27987cd2f0b966a4627, regionState=OPEN, openSeqNum=5, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:58:55,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-12-05T11:58:55,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 in 182 msec 2024-12-05T11:58:55,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-05T11:58:55,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, REOPEN/MOVE in 520 msec 2024-12-05T11:58:55,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-05T11:58:55,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-12-05T11:58:55,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 949 msec 2024-12-05T11:58:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-05T11:58:55,536 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:55,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,537 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:55,537 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:55,538 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cadf4d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:55,540 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30d933c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:55,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:55,543 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ece7c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:55,545 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@287f8123, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:55,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:55,547 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514e9fe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:55,549 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:55,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37e64342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:55,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:55,552 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bee5e81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:55,554 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300d7ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:55,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:55,557 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461196dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:55,559 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63de7c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:55,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:55,561 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ddcb63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:55,562 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c20471e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:55,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:55,565 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5087874a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:55,566 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4247d9c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:55,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:55,568 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5768676e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:55,570 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a1a52f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:55,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:55,572 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e331ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:55,573 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b636769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a2803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,33741,-1 for getting cluster id 2024-12-05T11:58:55,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:58:55,574 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffd85ce0-e522-4ae9-bc47-bc214516dd87' 2024-12-05T11:58:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:58:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffd85ce0-e522-4ae9-bc47-bc214516dd87" 2024-12-05T11:58:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a46e558, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,33741,-1] 2024-12-05T11:58:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:58:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:58:55,575 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:58:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd79254, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:58:55,576 DEBUG [Thread-2393 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:55,576 DEBUG [Thread-2389 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:55,576 DEBUG [Thread-2395 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:55,576 DEBUG [Thread-2386 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:55,577 DEBUG [Thread-2380 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:55,578 DEBUG [Thread-2397 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:55,578 DEBUG [Thread-2378 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:55,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:55,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:55,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:55,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:55,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:55,579 DEBUG [Thread-2384 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:55,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-05T11:58:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:55,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:58:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T11:58:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:58:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:58:55,581 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:58:55,582 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:55,582 DEBUG [Thread-2378 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:55,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:55,583 DEBUG [Thread-2378 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:55,583 DEBUG [Thread-2382 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:55,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:55,584 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:58:55,585 DEBUG [Thread-2391 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:58:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:58:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:58:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40121,1733399769572, seqNum=-1] 2024-12-05T11:58:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:58:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:58:55,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:55,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:58:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:58:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5] 2024-12-05T11:58:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,610 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:55,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T11:58:55,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056c2101c45ce147c2a67e7d9da68555ee_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399935609/Put/seqid=0 2024-12-05T11:58:55,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733399995638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733399995639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733399995640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399995639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733399995641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742357_1533 (size=12154) 2024-12-05T11:58:55,669 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:55,673 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056c2101c45ce147c2a67e7d9da68555ee_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056c2101c45ce147c2a67e7d9da68555ee_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:55,673 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/e23ad39c4a624da0af19dde105c5a21d, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:58:55,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/e23ad39c4a624da0af19dde105c5a21d is 175, key is test_row_0/A:col10/1733399935609/Put/seqid=0 2024-12-05T11:58:55,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742358_1534 (size=30955) 2024-12-05T11:58:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T11:58:55,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T11:58:55,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:55,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:58:55,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:55,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:55,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:55,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733399995748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733399995748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733399995748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399995748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733399995748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T11:58:55,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:55,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:58:55,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:55,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:55,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T11:58:55,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733399995958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733399995958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733399995958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399995958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733399995958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T11:58:56,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:56,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:58:56,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:56,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:56,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:56,100 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/e23ad39c4a624da0af19dde105c5a21d 2024-12-05T11:58:56,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/2d624c0f22dd4683833eb074e4f3403b is 50, key is test_row_0/B:col10/1733399935609/Put/seqid=0 2024-12-05T11:58:56,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742359_1535 (size=12001) 2024-12-05T11:58:56,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T11:58:56,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:56,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:58:56,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:56,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:56,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T11:58:56,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733399996268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733399996268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733399996268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399996268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733399996268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T11:58:56,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:56,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:58:56,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:56,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:56,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:56,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T11:58:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:58:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:56,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:56,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/2d624c0f22dd4683833eb074e4f3403b 2024-12-05T11:58:56,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/2409820fa7f045e1a46ea35c77075949 is 50, key is test_row_0/C:col10/1733399935609/Put/seqid=0 2024-12-05T11:58:56,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742360_1536 (size=12001) 2024-12-05T11:58:56,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/2409820fa7f045e1a46ea35c77075949 2024-12-05T11:58:56,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/e23ad39c4a624da0af19dde105c5a21d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e23ad39c4a624da0af19dde105c5a21d 2024-12-05T11:58:56,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e23ad39c4a624da0af19dde105c5a21d, entries=150, sequenceid=15, filesize=30.2 K 2024-12-05T11:58:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/2d624c0f22dd4683833eb074e4f3403b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/2d624c0f22dd4683833eb074e4f3403b 2024-12-05T11:58:56,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/2d624c0f22dd4683833eb074e4f3403b, entries=150, sequenceid=15, filesize=11.7 K 2024-12-05T11:58:56,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/2409820fa7f045e1a46ea35c77075949 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/2409820fa7f045e1a46ea35c77075949 2024-12-05T11:58:56,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/2409820fa7f045e1a46ea35c77075949, entries=150, sequenceid=15, filesize=11.7 K 2024-12-05T11:58:56,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 990ce2fe1eb6f27987cd2f0b966a4627 in 972ms, sequenceid=15, compaction requested=false 2024-12-05T11:58:56,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:58:56,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T11:58:56,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:56,655 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-05T11:58:56,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055ce66a940c6f49c984785c8669382bd7_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399935636/Put/seqid=0 2024-12-05T11:58:56,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742361_1537 (size=12154) 2024-12-05T11:58:56,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:56,685 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055ce66a940c6f49c984785c8669382bd7_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055ce66a940c6f49c984785c8669382bd7_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:56,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/8bbdf99191a840abb75c1186814dd657, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:58:56,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/8bbdf99191a840abb75c1186814dd657 is 175, key is test_row_0/A:col10/1733399935636/Put/seqid=0 2024-12-05T11:58:56,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742362_1538 (size=30955) 2024-12-05T11:58:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T11:58:56,718 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/8bbdf99191a840abb75c1186814dd657 2024-12-05T11:58:56,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/09e6b800ce4f4ba793e6b9df7926a25c is 50, key is test_row_0/B:col10/1733399935636/Put/seqid=0 2024-12-05T11:58:56,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742363_1539 (size=12001) 2024-12-05T11:58:56,741 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/09e6b800ce4f4ba793e6b9df7926a25c 2024-12-05T11:58:56,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/6d3fbf92042643b8a08566dcca4dfc91 is 50, key is test_row_0/C:col10/1733399935636/Put/seqid=0 2024-12-05T11:58:56,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742364_1540 (size=12001) 2024-12-05T11:58:56,774 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/6d3fbf92042643b8a08566dcca4dfc91 2024-12-05T11:58:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:56,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:58:56,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/8bbdf99191a840abb75c1186814dd657 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8bbdf99191a840abb75c1186814dd657 2024-12-05T11:58:56,784 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8bbdf99191a840abb75c1186814dd657, entries=150, sequenceid=41, filesize=30.2 K 2024-12-05T11:58:56,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/09e6b800ce4f4ba793e6b9df7926a25c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/09e6b800ce4f4ba793e6b9df7926a25c 2024-12-05T11:58:56,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733399996785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,790 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/09e6b800ce4f4ba793e6b9df7926a25c, entries=150, sequenceid=41, filesize=11.7 K 2024-12-05T11:58:56,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733399996786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733399996786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733399996786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399996787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/6d3fbf92042643b8a08566dcca4dfc91 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/6d3fbf92042643b8a08566dcca4dfc91 2024-12-05T11:58:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,801 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/6d3fbf92042643b8a08566dcca4dfc91, entries=150, sequenceid=41, filesize=11.7 K 2024-12-05T11:58:56,802 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 990ce2fe1eb6f27987cd2f0b966a4627 in 147ms, sequenceid=41, compaction requested=false 2024-12-05T11:58:56,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:58:56,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:56,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-05T11:58:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-05T11:58:56,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-05T11:58:56,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2200 sec 2024-12-05T11:58:56,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.2270 sec 2024-12-05T11:58:56,899 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:56,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T11:58:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:56,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c0918d906db14f4e91194ba2849d5303_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399936784/Put/seqid=0 2024-12-05T11:58:56,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742365_1541 (size=14594) 2024-12-05T11:58:56,932 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:56,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733399996927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733399996927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733399996928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733399996929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:56,935 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c0918d906db14f4e91194ba2849d5303_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c0918d906db14f4e91194ba2849d5303_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:56,936 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/78deb9bb1041463f85c4fff391429cba, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:58:56,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/78deb9bb1041463f85c4fff391429cba is 175, key is test_row_0/A:col10/1733399936784/Put/seqid=0 2024-12-05T11:58:56,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742366_1542 (size=39549) 2024-12-05T11:58:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733399997038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733399997038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733399997038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733399997038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733399997248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733399997248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733399997248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733399997248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,340 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/78deb9bb1041463f85c4fff391429cba 2024-12-05T11:58:57,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/d3fde9e9774b4b55ab2ffee7833a9634 is 50, key is test_row_0/B:col10/1733399936784/Put/seqid=0 2024-12-05T11:58:57,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742367_1543 (size=12001) 2024-12-05T11:58:57,424 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:58:57,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733399997557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733399997557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733399997558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733399997558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T11:58:57,728 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:57,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-05T11:58:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T11:58:57,732 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:57,733 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:57,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:57,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/d3fde9e9774b4b55ab2ffee7833a9634 2024-12-05T11:58:57,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/e26b4838f21c4417a589010d3877c60d is 50, key is test_row_0/C:col10/1733399936784/Put/seqid=0 2024-12-05T11:58:57,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742368_1544 (size=12001) 2024-12-05T11:58:57,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/e26b4838f21c4417a589010d3877c60d 2024-12-05T11:58:57,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/78deb9bb1041463f85c4fff391429cba as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/78deb9bb1041463f85c4fff391429cba 2024-12-05T11:58:57,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/78deb9bb1041463f85c4fff391429cba, entries=200, sequenceid=55, filesize=38.6 K 2024-12-05T11:58:57,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/d3fde9e9774b4b55ab2ffee7833a9634 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/d3fde9e9774b4b55ab2ffee7833a9634 2024-12-05T11:58:57,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/d3fde9e9774b4b55ab2ffee7833a9634, entries=150, sequenceid=55, filesize=11.7 K 2024-12-05T11:58:57,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/e26b4838f21c4417a589010d3877c60d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/e26b4838f21c4417a589010d3877c60d 2024-12-05T11:58:57,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/e26b4838f21c4417a589010d3877c60d, entries=150, sequenceid=55, filesize=11.7 K 2024-12-05T11:58:57,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 990ce2fe1eb6f27987cd2f0b966a4627 in 884ms, sequenceid=55, compaction requested=true 2024-12-05T11:58:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:58:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:58:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:57,784 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:57,784 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:58:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:58:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:57,785 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:57,785 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/A is initiating minor compaction (all files) 2024-12-05T11:58:57,785 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/A in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:57,785 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e23ad39c4a624da0af19dde105c5a21d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8bbdf99191a840abb75c1186814dd657, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/78deb9bb1041463f85c4fff391429cba] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=99.1 K 2024-12-05T11:58:57,785 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:57,785 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e23ad39c4a624da0af19dde105c5a21d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8bbdf99191a840abb75c1186814dd657, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/78deb9bb1041463f85c4fff391429cba] 2024-12-05T11:58:57,786 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:57,786 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/B is initiating minor compaction (all files) 2024-12-05T11:58:57,786 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/B in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:57,786 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/2d624c0f22dd4683833eb074e4f3403b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/09e6b800ce4f4ba793e6b9df7926a25c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/d3fde9e9774b4b55ab2ffee7833a9634] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=35.2 K 2024-12-05T11:58:57,786 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e23ad39c4a624da0af19dde105c5a21d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733399935609 2024-12-05T11:58:57,786 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bbdf99191a840abb75c1186814dd657, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733399935636 2024-12-05T11:58:57,786 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d624c0f22dd4683833eb074e4f3403b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733399935609 2024-12-05T11:58:57,787 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 09e6b800ce4f4ba793e6b9df7926a25c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733399935636 2024-12-05T11:58:57,787 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78deb9bb1041463f85c4fff391429cba, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733399936780 2024-12-05T11:58:57,787 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d3fde9e9774b4b55ab2ffee7833a9634, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733399936784 2024-12-05T11:58:57,795 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#B#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:57,796 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/14128a2eb3ad4d81b15d7e79d345593b is 50, key is test_row_0/B:col10/1733399936784/Put/seqid=0 2024-12-05T11:58:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:57,801 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:57,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T11:58:57,807 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:58:57,810 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412054590d3a15b044e7eaf9ff3b358c24b4e_990ce2fe1eb6f27987cd2f0b966a4627 store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:58:57,811 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412054590d3a15b044e7eaf9ff3b358c24b4e_990ce2fe1eb6f27987cd2f0b966a4627, store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:58:57,811 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054590d3a15b044e7eaf9ff3b358c24b4e_990ce2fe1eb6f27987cd2f0b966a4627 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:58:57,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205113676bffe0948f89f50bd3e47fbc3f2_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399936921/Put/seqid=0 2024-12-05T11:58:57,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742369_1545 (size=12104) 2024-12-05T11:58:57,836 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/14128a2eb3ad4d81b15d7e79d345593b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/14128a2eb3ad4d81b15d7e79d345593b 2024-12-05T11:58:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T11:58:57,841 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/B of 990ce2fe1eb6f27987cd2f0b966a4627 into 14128a2eb3ad4d81b15d7e79d345593b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:57,841 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:58:57,841 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/B, priority=13, startTime=1733399937784; duration=0sec 2024-12-05T11:58:57,841 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:58:57,841 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:B 2024-12-05T11:58:57,841 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:58:57,842 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:58:57,843 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/C is initiating minor compaction (all files) 2024-12-05T11:58:57,843 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/C in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:57,843 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/2409820fa7f045e1a46ea35c77075949, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/6d3fbf92042643b8a08566dcca4dfc91, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/e26b4838f21c4417a589010d3877c60d] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=35.2 K 2024-12-05T11:58:57,843 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2409820fa7f045e1a46ea35c77075949, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733399935609 2024-12-05T11:58:57,843 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d3fbf92042643b8a08566dcca4dfc91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733399935636 2024-12-05T11:58:57,844 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting e26b4838f21c4417a589010d3877c60d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733399936784 2024-12-05T11:58:57,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399997847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:57,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742370_1546 (size=4469) 2024-12-05T11:58:57,853 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#A#compaction#467 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:57,853 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/8ef0ef2530c14eca9d54d31dba0b84ab is 175, key is test_row_0/A:col10/1733399936784/Put/seqid=0 2024-12-05T11:58:57,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742371_1547 (size=12154) 2024-12-05T11:58:57,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:57,872 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205113676bffe0948f89f50bd3e47fbc3f2_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205113676bffe0948f89f50bd3e47fbc3f2_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:57,872 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/30e1ee2cd5d44283a41af23da498295d, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:58:57,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/30e1ee2cd5d44283a41af23da498295d is 175, key is test_row_0/A:col10/1733399936921/Put/seqid=0 2024-12-05T11:58:57,876 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:58:57,876 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/dd33f64d81d041dbbefa05b6da4b60f0 is 50, key is test_row_0/C:col10/1733399936784/Put/seqid=0 2024-12-05T11:58:57,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742372_1548 (size=31058) 2024-12-05T11:58:57,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742374_1550 (size=12104) 2024-12-05T11:58:57,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T11:58:57,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:57,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:58:57,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:57,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:57,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:57,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742373_1549 (size=30955) 2024-12-05T11:58:57,890 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/dd33f64d81d041dbbefa05b6da4b60f0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/dd33f64d81d041dbbefa05b6da4b60f0 2024-12-05T11:58:57,890 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/30e1ee2cd5d44283a41af23da498295d 2024-12-05T11:58:57,894 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/C of 990ce2fe1eb6f27987cd2f0b966a4627 into dd33f64d81d041dbbefa05b6da4b60f0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:57,894 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:58:57,894 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/C, priority=13, startTime=1733399937784; duration=0sec 2024-12-05T11:58:57,895 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:57,895 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:C 2024-12-05T11:58:57,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/b7ee54576da644d3bd8c85ec47744293 is 50, key is test_row_0/B:col10/1733399936921/Put/seqid=0 2024-12-05T11:58:57,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742375_1551 (size=12001) 2024-12-05T11:58:57,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:57,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399997958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:57,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:57,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:58,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T11:58:58,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:58,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:58:58,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:58,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:58,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T11:58:58,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733399998068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:58,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733399998068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:58,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733399998068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:58,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733399998068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:58,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:58,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399998168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:58,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T11:58:58,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:58,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:58:58,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:58,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:58,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:58:58,286 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/8ef0ef2530c14eca9d54d31dba0b84ab as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8ef0ef2530c14eca9d54d31dba0b84ab 2024-12-05T11:58:58,290 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/A of 990ce2fe1eb6f27987cd2f0b966a4627 into 8ef0ef2530c14eca9d54d31dba0b84ab(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:58:58,290 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:58:58,290 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/A, priority=13, startTime=1733399937784; duration=0sec 2024-12-05T11:58:58,290 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:58:58,290 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:A 2024-12-05T11:58:58,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/b7ee54576da644d3bd8c85ec47744293 2024-12-05T11:58:58,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/b802f4b28b054183994f3c34ec845c84 is 50, key is test_row_0/C:col10/1733399936921/Put/seqid=0 2024-12-05T11:58:58,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742376_1552 (size=12001) 2024-12-05T11:58:58,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/b802f4b28b054183994f3c34ec845c84 2024-12-05T11:58:58,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/30e1ee2cd5d44283a41af23da498295d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/30e1ee2cd5d44283a41af23da498295d 2024-12-05T11:58:58,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/30e1ee2cd5d44283a41af23da498295d, entries=150, sequenceid=78, filesize=30.2 K 2024-12-05T11:58:58,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/b7ee54576da644d3bd8c85ec47744293 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b7ee54576da644d3bd8c85ec47744293 2024-12-05T11:58:58,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b7ee54576da644d3bd8c85ec47744293, entries=150, sequenceid=78, filesize=11.7 K 2024-12-05T11:58:58,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/b802f4b28b054183994f3c34ec845c84 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/b802f4b28b054183994f3c34ec845c84 2024-12-05T11:58:58,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/b802f4b28b054183994f3c34ec845c84, entries=150, sequenceid=78, filesize=11.7 K 2024-12-05T11:58:58,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 990ce2fe1eb6f27987cd2f0b966a4627 in 526ms, sequenceid=78, compaction requested=false 2024-12-05T11:58:58,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:58:58,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T11:58:58,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:58,343 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T11:58:58,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057532d62d62164c3486fbe07e80e2106c_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399937845/Put/seqid=0 2024-12-05T11:58:58,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742377_1553 (size=12154) 2024-12-05T11:58:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T11:58:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:58,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:58:58,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399998544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:58,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399998658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:58,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:58:58,756 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057532d62d62164c3486fbe07e80e2106c_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057532d62d62164c3486fbe07e80e2106c_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:58,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/781552fdd0e7414499dc0b71f9b0fb96, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:58:58,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/781552fdd0e7414499dc0b71f9b0fb96 is 175, key is test_row_0/A:col10/1733399937845/Put/seqid=0 2024-12-05T11:58:58,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742378_1554 (size=30955) 2024-12-05T11:58:58,760 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/781552fdd0e7414499dc0b71f9b0fb96 2024-12-05T11:58:58,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/db788f00c3204176adf8f96db4d832ad is 50, key is test_row_0/B:col10/1733399937845/Put/seqid=0 2024-12-05T11:58:58,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742379_1555 (size=12001) 2024-12-05T11:58:58,780 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/db788f00c3204176adf8f96db4d832ad 2024-12-05T11:58:58,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/c2f11cd7907a469fa3998b03832ffa9e is 50, key is test_row_0/C:col10/1733399937845/Put/seqid=0 2024-12-05T11:58:58,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742380_1556 (size=12001) 2024-12-05T11:58:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T11:58:58,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399998868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:58,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:58,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:59,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733399999078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:59,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:59,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733399999078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733399999078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:59,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733399999078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:59,080 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:59,080 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:59,080 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:59,081 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:58:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:59,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:59,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399999177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:59,196 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/c2f11cd7907a469fa3998b03832ffa9e 2024-12-05T11:58:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/781552fdd0e7414499dc0b71f9b0fb96 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/781552fdd0e7414499dc0b71f9b0fb96 2024-12-05T11:58:59,203 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/781552fdd0e7414499dc0b71f9b0fb96, entries=150, sequenceid=94, filesize=30.2 K 2024-12-05T11:58:59,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/db788f00c3204176adf8f96db4d832ad as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/db788f00c3204176adf8f96db4d832ad 2024-12-05T11:58:59,206 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/db788f00c3204176adf8f96db4d832ad, entries=150, sequenceid=94, filesize=11.7 K 2024-12-05T11:58:59,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/c2f11cd7907a469fa3998b03832ffa9e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/c2f11cd7907a469fa3998b03832ffa9e 2024-12-05T11:58:59,210 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/c2f11cd7907a469fa3998b03832ffa9e, entries=150, sequenceid=94, filesize=11.7 K 2024-12-05T11:58:59,210 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 990ce2fe1eb6f27987cd2f0b966a4627 in 867ms, sequenceid=94, compaction requested=true 2024-12-05T11:58:59,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:58:59,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:58:59,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-05T11:58:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-05T11:58:59,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-05T11:58:59,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-12-05T11:58:59,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4840 sec 2024-12-05T11:58:59,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:58:59,690 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:58:59,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T11:58:59,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ba0389c0aced44ffaf72df7a85d47e9a_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399938509/Put/seqid=0 2024-12-05T11:58:59,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742381_1557 (size=12154) 2024-12-05T11:58:59,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399999719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:59,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:58:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733399999828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:58:59,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:58:59,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:58:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T11:58:59,878 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:58:59,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:58:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-05T11:58:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T11:58:59,880 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:58:59,880 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:58:59,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:58:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T11:59:00,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T11:59:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400000038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:00,098 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:00,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ba0389c0aced44ffaf72df7a85d47e9a_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ba0389c0aced44ffaf72df7a85d47e9a_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:00,102 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/5c4e8bd2180b414f8e23de3de6e24a66, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:00,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/5c4e8bd2180b414f8e23de3de6e24a66 is 175, key is test_row_0/A:col10/1733399938509/Put/seqid=0 2024-12-05T11:59:00,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742382_1558 (size=30955) 2024-12-05T11:59:00,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T11:59:00,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:00,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T11:59:00,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T11:59:00,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:00,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400000348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:00,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T11:59:00,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:00,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,506 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/5c4e8bd2180b414f8e23de3de6e24a66 2024-12-05T11:59:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T11:59:00,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/ac8e082d4ebe4856a956605b8c6b8f26 is 50, key is test_row_0/B:col10/1733399938509/Put/seqid=0 2024-12-05T11:59:00,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742383_1559 (size=12001) 2024-12-05T11:59:00,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T11:59:00,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:00,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T11:59:00,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:00,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:00,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400000858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:00,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/ac8e082d4ebe4856a956605b8c6b8f26 2024-12-05T11:59:00,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/cb3894d0de4d4b6e89f827fa6805c0cc is 50, key is test_row_0/C:col10/1733399938509/Put/seqid=0 2024-12-05T11:59:00,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742384_1560 (size=12001) 2024-12-05T11:59:00,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/cb3894d0de4d4b6e89f827fa6805c0cc 2024-12-05T11:59:00,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/5c4e8bd2180b414f8e23de3de6e24a66 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/5c4e8bd2180b414f8e23de3de6e24a66 2024-12-05T11:59:00,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/5c4e8bd2180b414f8e23de3de6e24a66, entries=150, sequenceid=118, filesize=30.2 K 2024-12-05T11:59:00,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/ac8e082d4ebe4856a956605b8c6b8f26 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/ac8e082d4ebe4856a956605b8c6b8f26 2024-12-05T11:59:00,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/ac8e082d4ebe4856a956605b8c6b8f26, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T11:59:00,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/cb3894d0de4d4b6e89f827fa6805c0cc as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/cb3894d0de4d4b6e89f827fa6805c0cc 2024-12-05T11:59:00,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/cb3894d0de4d4b6e89f827fa6805c0cc, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T11:59:00,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 990ce2fe1eb6f27987cd2f0b966a4627 in 1251ms, sequenceid=118, compaction requested=true 2024-12-05T11:59:00,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:59:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:00,942 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:59:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:59:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:00,942 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:59:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:59:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:59:00,942 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123923 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:59:00,942 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/A is initiating minor compaction (all files) 2024-12-05T11:59:00,942 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:59:00,943 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/B is initiating minor compaction (all files) 2024-12-05T11:59:00,943 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/A in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,943 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/B in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,943 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8ef0ef2530c14eca9d54d31dba0b84ab, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/30e1ee2cd5d44283a41af23da498295d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/781552fdd0e7414499dc0b71f9b0fb96, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/5c4e8bd2180b414f8e23de3de6e24a66] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=121.0 K 2024-12-05T11:59:00,943 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/14128a2eb3ad4d81b15d7e79d345593b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b7ee54576da644d3bd8c85ec47744293, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/db788f00c3204176adf8f96db4d832ad, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/ac8e082d4ebe4856a956605b8c6b8f26] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=47.0 K 2024-12-05T11:59:00,943 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,943 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8ef0ef2530c14eca9d54d31dba0b84ab, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/30e1ee2cd5d44283a41af23da498295d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/781552fdd0e7414499dc0b71f9b0fb96, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/5c4e8bd2180b414f8e23de3de6e24a66] 2024-12-05T11:59:00,943 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 14128a2eb3ad4d81b15d7e79d345593b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733399936784 2024-12-05T11:59:00,943 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ef0ef2530c14eca9d54d31dba0b84ab, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733399936784 2024-12-05T11:59:00,943 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting b7ee54576da644d3bd8c85ec47744293, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733399936921 2024-12-05T11:59:00,943 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30e1ee2cd5d44283a41af23da498295d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733399936921 2024-12-05T11:59:00,943 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting db788f00c3204176adf8f96db4d832ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399937813 2024-12-05T11:59:00,944 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 781552fdd0e7414499dc0b71f9b0fb96, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399937813 2024-12-05T11:59:00,944 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ac8e082d4ebe4856a956605b8c6b8f26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733399938509 2024-12-05T11:59:00,944 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4e8bd2180b414f8e23de3de6e24a66, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733399938509 2024-12-05T11:59:00,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T11:59:00,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:00,948 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T11:59:00,948 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:00,951 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#B#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:00,951 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/24e90f1561ff459b829d9e21371b8abb is 50, key is test_row_0/B:col10/1733399938509/Put/seqid=0 2024-12-05T11:59:00,952 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120563006205ef0448bda67918cce8c13383_990ce2fe1eb6f27987cd2f0b966a4627 store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:00,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742385_1561 (size=12241) 2024-12-05T11:59:00,954 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120563006205ef0448bda67918cce8c13383_990ce2fe1eb6f27987cd2f0b966a4627, store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:00,955 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120563006205ef0448bda67918cce8c13383_990ce2fe1eb6f27987cd2f0b966a4627 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:00,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f89b11549d0b428da92d85515c51e4da_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399939706/Put/seqid=0 2024-12-05T11:59:00,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742386_1562 (size=4469) 2024-12-05T11:59:00,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742387_1563 (size=12204) 2024-12-05T11:59:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T11:59:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:01,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:01,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400001127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400001128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400001128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400001134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400001238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400001238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400001238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400001248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,359 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#A#compaction#478 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:01,360 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/c3498e67baf94e7293684d6d67371d6f is 175, key is test_row_0/A:col10/1733399938509/Put/seqid=0 2024-12-05T11:59:01,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:01,362 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/24e90f1561ff459b829d9e21371b8abb as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/24e90f1561ff459b829d9e21371b8abb 2024-12-05T11:59:01,369 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f89b11549d0b428da92d85515c51e4da_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f89b11549d0b428da92d85515c51e4da_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:01,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/02daf463d1034653baab0652719de198, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:01,377 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/B of 990ce2fe1eb6f27987cd2f0b966a4627 into 24e90f1561ff459b829d9e21371b8abb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:01,377 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:01,377 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/B, priority=12, startTime=1733399940942; duration=0sec 2024-12-05T11:59:01,377 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:59:01,377 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:B 2024-12-05T11:59:01,377 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:59:01,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/02daf463d1034653baab0652719de198 is 175, key is test_row_0/A:col10/1733399939706/Put/seqid=0 2024-12-05T11:59:01,380 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:59:01,380 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/C is initiating minor compaction (all files) 2024-12-05T11:59:01,380 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/C in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:01,380 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/dd33f64d81d041dbbefa05b6da4b60f0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/b802f4b28b054183994f3c34ec845c84, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/c2f11cd7907a469fa3998b03832ffa9e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/cb3894d0de4d4b6e89f827fa6805c0cc] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=47.0 K 2024-12-05T11:59:01,380 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting dd33f64d81d041dbbefa05b6da4b60f0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733399936784 2024-12-05T11:59:01,381 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting b802f4b28b054183994f3c34ec845c84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733399936921 2024-12-05T11:59:01,381 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting c2f11cd7907a469fa3998b03832ffa9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733399937813 2024-12-05T11:59:01,381 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting cb3894d0de4d4b6e89f827fa6805c0cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733399938509 2024-12-05T11:59:01,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742388_1564 (size=31195) 2024-12-05T11:59:01,440 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#C#compaction#481 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:01,441 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/650412bf4b3c4aaf9da84352c142d1d9 is 50, key is test_row_0/C:col10/1733399938509/Put/seqid=0 2024-12-05T11:59:01,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400001447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400001448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400001448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742389_1565 (size=31005) 2024-12-05T11:59:01,457 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/02daf463d1034653baab0652719de198 2024-12-05T11:59:01,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400001458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742390_1566 (size=12241) 2024-12-05T11:59:01,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/9ccebcb980884c0795d7923d9731b595 is 50, key is test_row_0/B:col10/1733399939706/Put/seqid=0 2024-12-05T11:59:01,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742391_1567 (size=12051) 2024-12-05T11:59:01,536 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/9ccebcb980884c0795d7923d9731b595 2024-12-05T11:59:01,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/bbd06002f4ff41cab1c38a4f9f317f88 is 50, key is test_row_0/C:col10/1733399939706/Put/seqid=0 2024-12-05T11:59:01,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742392_1568 (size=12051) 2024-12-05T11:59:01,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400001757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400001768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400001768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400001768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,819 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/c3498e67baf94e7293684d6d67371d6f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c3498e67baf94e7293684d6d67371d6f 2024-12-05T11:59:01,824 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/A of 990ce2fe1eb6f27987cd2f0b966a4627 into c3498e67baf94e7293684d6d67371d6f(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:01,824 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:01,824 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/A, priority=12, startTime=1733399940942; duration=0sec 2024-12-05T11:59:01,825 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:01,825 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:A 2024-12-05T11:59:01,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400001867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:01,871 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_1' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:59:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:01,891 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/650412bf4b3c4aaf9da84352c142d1d9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/650412bf4b3c4aaf9da84352c142d1d9 2024-12-05T11:59:01,895 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/C of 990ce2fe1eb6f27987cd2f0b966a4627 into 650412bf4b3c4aaf9da84352c142d1d9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:01,895 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:01,895 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/C, priority=12, startTime=1733399940942; duration=0sec 2024-12-05T11:59:01,895 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:01,895 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:C 2024-12-05T11:59:02,007 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/bbd06002f4ff41cab1c38a4f9f317f88 2024-12-05T11:59:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/02daf463d1034653baab0652719de198 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/02daf463d1034653baab0652719de198 2024-12-05T11:59:02,019 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/02daf463d1034653baab0652719de198, entries=150, sequenceid=130, filesize=30.3 K 2024-12-05T11:59:02,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/9ccebcb980884c0795d7923d9731b595 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/9ccebcb980884c0795d7923d9731b595 2024-12-05T11:59:02,026 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/9ccebcb980884c0795d7923d9731b595, entries=150, sequenceid=130, filesize=11.8 K 2024-12-05T11:59:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T11:59:02,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/bbd06002f4ff41cab1c38a4f9f317f88 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/bbd06002f4ff41cab1c38a4f9f317f88 2024-12-05T11:59:02,034 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/bbd06002f4ff41cab1c38a4f9f317f88, entries=150, sequenceid=130, filesize=11.8 K 2024-12-05T11:59:02,035 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 990ce2fe1eb6f27987cd2f0b966a4627 in 1087ms, sequenceid=130, compaction requested=false 2024-12-05T11:59:02,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:02,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:02,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-05T11:59:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-05T11:59:02,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-05T11:59:02,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1580 sec 2024-12-05T11:59:02,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.1710 sec 2024-12-05T11:59:02,273 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:59:02,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T11:59:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:02,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b90caa97ceea4ec18574cfe804d4bbde_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399941131/Put/seqid=0 2024-12-05T11:59:02,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400002290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400002293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400002296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400002297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742393_1569 (size=14794) 2024-12-05T11:59:02,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400002407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400002407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400002408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400002408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400002618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400002618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400002618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400002618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,728 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:02,734 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b90caa97ceea4ec18574cfe804d4bbde_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b90caa97ceea4ec18574cfe804d4bbde_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:02,736 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/0ec2d28892754cc6b7a4b256db2257bc, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:02,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/0ec2d28892754cc6b7a4b256db2257bc is 175, key is test_row_0/A:col10/1733399941131/Put/seqid=0 2024-12-05T11:59:02,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742394_1570 (size=39749) 2024-12-05T11:59:02,789 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/0ec2d28892754cc6b7a4b256db2257bc 2024-12-05T11:59:02,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/8a37bf95bba44cb5b6406a632b35fa7d is 50, key is test_row_0/B:col10/1733399941131/Put/seqid=0 2024-12-05T11:59:02,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742395_1571 (size=12151) 2024-12-05T11:59:02,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400002928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400002929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400002929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400002929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:03,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/8a37bf95bba44cb5b6406a632b35fa7d 2024-12-05T11:59:03,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/9df967d083a8470f98c255932b2c6524 is 50, key is test_row_0/C:col10/1733399941131/Put/seqid=0 2024-12-05T11:59:03,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742396_1572 (size=12151) 2024-12-05T11:59:03,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400003438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:03,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400003438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:03,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400003449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:03,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400003450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:03,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:03,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:03,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:03,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/9df967d083a8470f98c255932b2c6524 2024-12-05T11:59:03,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/0ec2d28892754cc6b7a4b256db2257bc as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0ec2d28892754cc6b7a4b256db2257bc 2024-12-05T11:59:03,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0ec2d28892754cc6b7a4b256db2257bc, entries=200, sequenceid=158, filesize=38.8 K 2024-12-05T11:59:03,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/8a37bf95bba44cb5b6406a632b35fa7d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/8a37bf95bba44cb5b6406a632b35fa7d 2024-12-05T11:59:03,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/8a37bf95bba44cb5b6406a632b35fa7d, entries=150, sequenceid=158, filesize=11.9 K 2024-12-05T11:59:03,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/9df967d083a8470f98c255932b2c6524 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/9df967d083a8470f98c255932b2c6524 2024-12-05T11:59:03,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/9df967d083a8470f98c255932b2c6524, entries=150, sequenceid=158, filesize=11.9 K 2024-12-05T11:59:03,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 990ce2fe1eb6f27987cd2f0b966a4627 in 1469ms, sequenceid=158, compaction requested=true 2024-12-05T11:59:03,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:03,742 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:59:03,743 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101949 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:59:03,743 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/A is initiating minor compaction (all files) 2024-12-05T11:59:03,743 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/A in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:03,743 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c3498e67baf94e7293684d6d67371d6f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/02daf463d1034653baab0652719de198, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0ec2d28892754cc6b7a4b256db2257bc] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=99.6 K 2024-12-05T11:59:03,743 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:03,743 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c3498e67baf94e7293684d6d67371d6f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/02daf463d1034653baab0652719de198, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0ec2d28892754cc6b7a4b256db2257bc] 2024-12-05T11:59:03,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:59:03,744 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3498e67baf94e7293684d6d67371d6f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733399938509 2024-12-05T11:59:03,744 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02daf463d1034653baab0652719de198, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733399939706 2024-12-05T11:59:03,744 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ec2d28892754cc6b7a4b256db2257bc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733399941126 2024-12-05T11:59:03,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:03,747 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:59:03,748 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:59:03,748 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/B is initiating minor compaction (all files) 2024-12-05T11:59:03,748 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/B in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:03,749 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/24e90f1561ff459b829d9e21371b8abb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/9ccebcb980884c0795d7923d9731b595, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/8a37bf95bba44cb5b6406a632b35fa7d] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=35.6 K 2024-12-05T11:59:03,750 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 24e90f1561ff459b829d9e21371b8abb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733399938509 2024-12-05T11:59:03,750 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ccebcb980884c0795d7923d9731b595, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733399939706 2024-12-05T11:59:03,751 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a37bf95bba44cb5b6406a632b35fa7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733399941127 2024-12-05T11:59:03,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:59:03,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:03,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:59:03,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:59:03,763 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:03,766 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#B#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:03,766 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/2744cfdd9e1c40cb808cd5e1bc7c61a4 is 50, key is test_row_0/B:col10/1733399941131/Put/seqid=0 2024-12-05T11:59:03,795 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412054afab546dc004d2f95805552bff0dcea_990ce2fe1eb6f27987cd2f0b966a4627 store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:03,798 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412054afab546dc004d2f95805552bff0dcea_990ce2fe1eb6f27987cd2f0b966a4627, store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:03,798 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054afab546dc004d2f95805552bff0dcea_990ce2fe1eb6f27987cd2f0b966a4627 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:03,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742397_1573 (size=12493) 2024-12-05T11:59:03,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742398_1574 (size=4469) 2024-12-05T11:59:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:03,892 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:59:03,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T11:59:03,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205131047b628a14f7982f782fd8d590dac_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399942291/Put/seqid=0 2024-12-05T11:59:03,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742399_1575 (size=14794) 2024-12-05T11:59:03,960 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:03,965 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205131047b628a14f7982f782fd8d590dac_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205131047b628a14f7982f782fd8d590dac_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:03,966 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/460ca6d138684386b1db8952c25857a2, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:03,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/460ca6d138684386b1db8952c25857a2 is 175, key is test_row_0/A:col10/1733399942291/Put/seqid=0 2024-12-05T11:59:03,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742400_1576 (size=39749) 2024-12-05T11:59:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T11:59:04,039 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:59:04,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:59:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-05T11:59:04,042 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:59:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T11:59:04,042 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:59:04,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:59:04,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400004065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T11:59:04,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400004178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:04,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T11:59:04,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:04,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:04,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:04,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:04,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:04,271 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/2744cfdd9e1c40cb808cd5e1bc7c61a4 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/2744cfdd9e1c40cb808cd5e1bc7c61a4 2024-12-05T11:59:04,277 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/B of 990ce2fe1eb6f27987cd2f0b966a4627 into 2744cfdd9e1c40cb808cd5e1bc7c61a4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:04,277 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:04,277 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/B, priority=13, startTime=1733399943747; duration=0sec 2024-12-05T11:59:04,277 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:59:04,277 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:B 2024-12-05T11:59:04,277 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:59:04,278 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#A#compaction#487 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:04,279 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/d7b786f664b64a159e713e8af6d73493 is 175, key is test_row_0/A:col10/1733399941131/Put/seqid=0 2024-12-05T11:59:04,279 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:59:04,279 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/C is initiating minor compaction (all files) 2024-12-05T11:59:04,279 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/C in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:04,279 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/650412bf4b3c4aaf9da84352c142d1d9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/bbd06002f4ff41cab1c38a4f9f317f88, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/9df967d083a8470f98c255932b2c6524] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=35.6 K 2024-12-05T11:59:04,280 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 650412bf4b3c4aaf9da84352c142d1d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733399938509 2024-12-05T11:59:04,280 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting bbd06002f4ff41cab1c38a4f9f317f88, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733399939706 2024-12-05T11:59:04,280 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9df967d083a8470f98c255932b2c6524, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733399941127 2024-12-05T11:59:04,300 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#C#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:04,301 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/3aee42c567954d96ad4e3703ceb6472e is 50, key is test_row_0/C:col10/1733399941131/Put/seqid=0 2024-12-05T11:59:04,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742401_1577 (size=31447) 2024-12-05T11:59:04,317 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/d7b786f664b64a159e713e8af6d73493 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d7b786f664b64a159e713e8af6d73493 2024-12-05T11:59:04,323 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/A of 990ce2fe1eb6f27987cd2f0b966a4627 into d7b786f664b64a159e713e8af6d73493(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:04,323 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:04,323 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/A, priority=13, startTime=1733399943742; duration=0sec 2024-12-05T11:59:04,323 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:04,323 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:A 2024-12-05T11:59:04,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742402_1578 (size=12493) 2024-12-05T11:59:04,346 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/3aee42c567954d96ad4e3703ceb6472e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3aee42c567954d96ad4e3703ceb6472e 2024-12-05T11:59:04,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T11:59:04,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:04,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:04,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:04,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:04,350 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/C of 990ce2fe1eb6f27987cd2f0b966a4627 into 3aee42c567954d96ad4e3703ceb6472e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:04,350 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:04,350 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/C, priority=13, startTime=1733399943752; duration=0sec 2024-12-05T11:59:04,350 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:04,350 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:C 2024-12-05T11:59:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T11:59:04,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:04,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400004388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:04,398 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/460ca6d138684386b1db8952c25857a2 2024-12-05T11:59:04,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/f1c7f3b3e47b4c4184b62aaf9b86a76c is 50, key is test_row_0/B:col10/1733399942291/Put/seqid=0 2024-12-05T11:59:04,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400004447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:04,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400004447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:04,449 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:59:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:04,450 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:59:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:04,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400004458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:04,460 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:59:04,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:04,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742403_1579 (size=12151) 2024-12-05T11:59:04,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/f1c7f3b3e47b4c4184b62aaf9b86a76c 2024-12-05T11:59:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400004468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:04,469 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:59:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:04,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/36ee47ed0fa84536a89c53154811270d is 50, key is test_row_0/C:col10/1733399942291/Put/seqid=0 2024-12-05T11:59:04,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T11:59:04,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:04,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:04,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:04,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:04,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:04,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742404_1580 (size=12151) 2024-12-05T11:59:04,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/36ee47ed0fa84536a89c53154811270d 2024-12-05T11:59:04,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/460ca6d138684386b1db8952c25857a2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/460ca6d138684386b1db8952c25857a2 2024-12-05T11:59:04,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/460ca6d138684386b1db8952c25857a2, entries=200, sequenceid=170, filesize=38.8 K 2024-12-05T11:59:04,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/f1c7f3b3e47b4c4184b62aaf9b86a76c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/f1c7f3b3e47b4c4184b62aaf9b86a76c 2024-12-05T11:59:04,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/f1c7f3b3e47b4c4184b62aaf9b86a76c, entries=150, sequenceid=170, filesize=11.9 K 2024-12-05T11:59:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/36ee47ed0fa84536a89c53154811270d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/36ee47ed0fa84536a89c53154811270d 2024-12-05T11:59:04,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/36ee47ed0fa84536a89c53154811270d, entries=150, sequenceid=170, filesize=11.9 K 2024-12-05T11:59:04,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 990ce2fe1eb6f27987cd2f0b966a4627 in 659ms, sequenceid=170, compaction requested=false 2024-12-05T11:59:04,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:04,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T11:59:04,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:04,656 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T11:59:04,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056957b2eb7b9a44d2a55afce462934b08_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399944054/Put/seqid=0 2024-12-05T11:59:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T11:59:04,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742405_1581 (size=12304) 2024-12-05T11:59:04,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:04,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:04,703 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056957b2eb7b9a44d2a55afce462934b08_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056957b2eb7b9a44d2a55afce462934b08_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:04,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/abb34a191ea04db3bc3af7cb18de9180, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:04,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/abb34a191ea04db3bc3af7cb18de9180 is 175, key is test_row_0/A:col10/1733399944054/Put/seqid=0 2024-12-05T11:59:04,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400004744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:04,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742406_1582 (size=31105) 2024-12-05T11:59:04,755 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/abb34a191ea04db3bc3af7cb18de9180 2024-12-05T11:59:04,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/ec2b5c3898e84137ba3ab47ecea3f6d0 is 50, key is test_row_0/B:col10/1733399944054/Put/seqid=0 2024-12-05T11:59:04,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742407_1583 (size=12151) 2024-12-05T11:59:04,815 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/ec2b5c3898e84137ba3ab47ecea3f6d0 2024-12-05T11:59:04,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/ae25df7e441a436aa57f96a53a247e45 is 50, key is test_row_0/C:col10/1733399944054/Put/seqid=0 2024-12-05T11:59:04,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400004858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:04,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742408_1584 (size=12151) 2024-12-05T11:59:05,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400005067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T11:59:05,286 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/ae25df7e441a436aa57f96a53a247e45 2024-12-05T11:59:05,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/abb34a191ea04db3bc3af7cb18de9180 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/abb34a191ea04db3bc3af7cb18de9180 2024-12-05T11:59:05,294 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/abb34a191ea04db3bc3af7cb18de9180, entries=150, sequenceid=197, filesize=30.4 K 2024-12-05T11:59:05,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/ec2b5c3898e84137ba3ab47ecea3f6d0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/ec2b5c3898e84137ba3ab47ecea3f6d0 2024-12-05T11:59:05,303 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/ec2b5c3898e84137ba3ab47ecea3f6d0, entries=150, sequenceid=197, filesize=11.9 K 2024-12-05T11:59:05,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/ae25df7e441a436aa57f96a53a247e45 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/ae25df7e441a436aa57f96a53a247e45 2024-12-05T11:59:05,307 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/ae25df7e441a436aa57f96a53a247e45, entries=150, sequenceid=197, filesize=11.9 K 2024-12-05T11:59:05,308 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 990ce2fe1eb6f27987cd2f0b966a4627 in 652ms, sequenceid=197, compaction requested=true 2024-12-05T11:59:05,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:05,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:05,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-05T11:59:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-05T11:59:05,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-05T11:59:05,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2670 sec 2024-12-05T11:59:05,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.2710 sec 2024-12-05T11:59:05,381 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:59:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:05,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T11:59:05,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059623b6f582f24263bc0fa82700017678_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399944709/Put/seqid=0 2024-12-05T11:59:05,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742409_1585 (size=14794) 2024-12-05T11:59:05,428 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:05,432 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059623b6f582f24263bc0fa82700017678_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059623b6f582f24263bc0fa82700017678_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:05,433 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/c623ae9f84734466835d219ae39dd277, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:05,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/c623ae9f84734466835d219ae39dd277 is 175, key is test_row_0/A:col10/1733399944709/Put/seqid=0 2024-12-05T11:59:05,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742410_1586 (size=39749) 2024-12-05T11:59:05,460 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/c623ae9f84734466835d219ae39dd277 2024-12-05T11:59:05,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/25d306dd2c4e4b11aea83b7f5dec0dc0 is 50, key is test_row_0/B:col10/1733399944709/Put/seqid=0 2024-12-05T11:59:05,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400005484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:05,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742411_1587 (size=12151) 2024-12-05T11:59:05,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/25d306dd2c4e4b11aea83b7f5dec0dc0 2024-12-05T11:59:05,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/d07520c664fb4e36a1ae0e4a5630ce97 is 50, key is test_row_0/C:col10/1733399944709/Put/seqid=0 2024-12-05T11:59:05,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742412_1588 (size=12151) 2024-12-05T11:59:05,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/d07520c664fb4e36a1ae0e4a5630ce97 2024-12-05T11:59:05,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/c623ae9f84734466835d219ae39dd277 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c623ae9f84734466835d219ae39dd277 2024-12-05T11:59:05,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c623ae9f84734466835d219ae39dd277, entries=200, sequenceid=210, filesize=38.8 K 2024-12-05T11:59:05,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/25d306dd2c4e4b11aea83b7f5dec0dc0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/25d306dd2c4e4b11aea83b7f5dec0dc0 2024-12-05T11:59:05,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/25d306dd2c4e4b11aea83b7f5dec0dc0, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T11:59:05,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/d07520c664fb4e36a1ae0e4a5630ce97 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/d07520c664fb4e36a1ae0e4a5630ce97 2024-12-05T11:59:05,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/d07520c664fb4e36a1ae0e4a5630ce97, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T11:59:05,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 990ce2fe1eb6f27987cd2f0b966a4627 in 187ms, sequenceid=210, compaction requested=true 2024-12-05T11:59:05,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:05,569 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:59:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:59:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:05,569 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:59:05,570 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:59:05,570 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/A is initiating minor compaction (all files) 2024-12-05T11:59:05,570 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/A in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:05,570 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d7b786f664b64a159e713e8af6d73493, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/460ca6d138684386b1db8952c25857a2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/abb34a191ea04db3bc3af7cb18de9180, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c623ae9f84734466835d219ae39dd277] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=138.7 K 2024-12-05T11:59:05,570 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:05,570 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d7b786f664b64a159e713e8af6d73493, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/460ca6d138684386b1db8952c25857a2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/abb34a191ea04db3bc3af7cb18de9180, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c623ae9f84734466835d219ae39dd277] 2024-12-05T11:59:05,570 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7b786f664b64a159e713e8af6d73493, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733399941127 2024-12-05T11:59:05,571 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 460ca6d138684386b1db8952c25857a2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733399942290 2024-12-05T11:59:05,571 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:59:05,571 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/B is initiating minor compaction (all files) 2024-12-05T11:59:05,571 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/B in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:05,571 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/2744cfdd9e1c40cb808cd5e1bc7c61a4, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/f1c7f3b3e47b4c4184b62aaf9b86a76c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/ec2b5c3898e84137ba3ab47ecea3f6d0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/25d306dd2c4e4b11aea83b7f5dec0dc0] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=47.8 K 2024-12-05T11:59:05,571 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting abb34a191ea04db3bc3af7cb18de9180, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733399944041 2024-12-05T11:59:05,572 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2744cfdd9e1c40cb808cd5e1bc7c61a4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733399941127 2024-12-05T11:59:05,572 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c623ae9f84734466835d219ae39dd277, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733399944709 2024-12-05T11:59:05,572 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting f1c7f3b3e47b4c4184b62aaf9b86a76c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733399942291 2024-12-05T11:59:05,573 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2b5c3898e84137ba3ab47ecea3f6d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733399944041 2024-12-05T11:59:05,573 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d306dd2c4e4b11aea83b7f5dec0dc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733399944709 2024-12-05T11:59:05,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:59:05,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:05,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:59:05,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:59:05,583 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#B#compaction#499 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:05,583 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:05,583 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/5f5b771bcbbc497aa798d53b895c705b is 50, key is test_row_0/B:col10/1733399944709/Put/seqid=0 2024-12-05T11:59:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:05,603 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:59:05,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T11:59:05,604 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205d4cf6402d85d4d4b90d7a98a38f82e7b_990ce2fe1eb6f27987cd2f0b966a4627 store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:05,607 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205d4cf6402d85d4d4b90d7a98a38f82e7b_990ce2fe1eb6f27987cd2f0b966a4627, store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:05,607 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d4cf6402d85d4d4b90d7a98a38f82e7b_990ce2fe1eb6f27987cd2f0b966a4627 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:05,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742413_1589 (size=12629) 2024-12-05T11:59:05,615 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/5f5b771bcbbc497aa798d53b895c705b as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/5f5b771bcbbc497aa798d53b895c705b 2024-12-05T11:59:05,623 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/B of 990ce2fe1eb6f27987cd2f0b966a4627 into 5f5b771bcbbc497aa798d53b895c705b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:05,623 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:05,623 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/B, priority=12, startTime=1733399945569; duration=0sec 2024-12-05T11:59:05,623 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:59:05,623 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:B 2024-12-05T11:59:05,624 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:59:05,626 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:59:05,626 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/C is initiating minor compaction (all files) 2024-12-05T11:59:05,626 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/C in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:05,626 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3aee42c567954d96ad4e3703ceb6472e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/36ee47ed0fa84536a89c53154811270d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/ae25df7e441a436aa57f96a53a247e45, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/d07520c664fb4e36a1ae0e4a5630ce97] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=47.8 K 2024-12-05T11:59:05,628 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aee42c567954d96ad4e3703ceb6472e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733399941127 2024-12-05T11:59:05,629 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ee47ed0fa84536a89c53154811270d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733399942291 2024-12-05T11:59:05,630 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting ae25df7e441a436aa57f96a53a247e45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733399944041 2024-12-05T11:59:05,631 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting d07520c664fb4e36a1ae0e4a5630ce97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733399944709 2024-12-05T11:59:05,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205eccc2e4cee1e49059ae8dfe9aeedb93b_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399945483/Put/seqid=0 2024-12-05T11:59:05,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400005657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:05,669 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#C#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:05,669 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/1cafb44330104195b2bba9e8e6180a0f is 50, key is test_row_0/C:col10/1733399944709/Put/seqid=0 2024-12-05T11:59:05,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742414_1590 (size=4469) 2024-12-05T11:59:05,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742415_1591 (size=14794) 2024-12-05T11:59:05,716 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:05,720 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205eccc2e4cee1e49059ae8dfe9aeedb93b_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205eccc2e4cee1e49059ae8dfe9aeedb93b_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:05,723 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/bfa4140e5fa24810984912b24f182298, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:05,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/bfa4140e5fa24810984912b24f182298 is 175, key is test_row_0/A:col10/1733399945483/Put/seqid=0 2024-12-05T11:59:05,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742416_1592 (size=12629) 2024-12-05T11:59:05,752 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/1cafb44330104195b2bba9e8e6180a0f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/1cafb44330104195b2bba9e8e6180a0f 2024-12-05T11:59:05,759 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/C of 990ce2fe1eb6f27987cd2f0b966a4627 into 1cafb44330104195b2bba9e8e6180a0f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:05,759 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:05,759 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/C, priority=12, startTime=1733399945576; duration=0sec 2024-12-05T11:59:05,759 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:05,759 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:C 2024-12-05T11:59:05,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400005768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:05,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:05,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:05,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:05,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742417_1593 (size=39749) 2024-12-05T11:59:05,780 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/bfa4140e5fa24810984912b24f182298 2024-12-05T11:59:05,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/e36b1f2730cb4cfcaedca6abdcc581b6 is 50, key is test_row_0/B:col10/1733399945483/Put/seqid=0 2024-12-05T11:59:05,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742418_1594 (size=12151) 2024-12-05T11:59:05,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/e36b1f2730cb4cfcaedca6abdcc581b6 2024-12-05T11:59:05,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/3da0aca2147040a79ed312a3673ff8ff is 50, key is test_row_0/C:col10/1733399945483/Put/seqid=0 2024-12-05T11:59:05,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742419_1595 (size=12151) 2024-12-05T11:59:05,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/3da0aca2147040a79ed312a3673ff8ff 2024-12-05T11:59:05,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/bfa4140e5fa24810984912b24f182298 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/bfa4140e5fa24810984912b24f182298 2024-12-05T11:59:05,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/bfa4140e5fa24810984912b24f182298, entries=200, sequenceid=234, filesize=38.8 K 2024-12-05T11:59:05,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/e36b1f2730cb4cfcaedca6abdcc581b6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/e36b1f2730cb4cfcaedca6abdcc581b6 2024-12-05T11:59:05,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/e36b1f2730cb4cfcaedca6abdcc581b6, entries=150, sequenceid=234, filesize=11.9 K 2024-12-05T11:59:05,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/3da0aca2147040a79ed312a3673ff8ff as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3da0aca2147040a79ed312a3673ff8ff 2024-12-05T11:59:05,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3da0aca2147040a79ed312a3673ff8ff, entries=150, sequenceid=234, filesize=11.9 K 2024-12-05T11:59:05,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 990ce2fe1eb6f27987cd2f0b966a4627 in 326ms, sequenceid=234, compaction requested=false 2024-12-05T11:59:05,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:05,980 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:59:05,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T11:59:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:06,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b703bc27ceba40c1abfb1a708ce17451_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399945638/Put/seqid=0 2024-12-05T11:59:06,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742420_1596 (size=12304) 2024-12-05T11:59:06,040 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:06,044 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b703bc27ceba40c1abfb1a708ce17451_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b703bc27ceba40c1abfb1a708ce17451_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:06,045 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/d862bb3eaf4e4046b4048ab9b54c4864, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:06,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/d862bb3eaf4e4046b4048ab9b54c4864 is 175, key is test_row_0/A:col10/1733399945638/Put/seqid=0 2024-12-05T11:59:06,110 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#A#compaction#500 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:06,110 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/2ddb0301117a43fe9e619087cface9c1 is 175, key is test_row_0/A:col10/1733399944709/Put/seqid=0 2024-12-05T11:59:06,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742421_1597 (size=31105) 2024-12-05T11:59:06,121 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/d862bb3eaf4e4046b4048ab9b54c4864 2024-12-05T11:59:06,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/b4b4103641e348e9bf2d6368f1b7cde6 is 50, key is test_row_0/B:col10/1733399945638/Put/seqid=0 2024-12-05T11:59:06,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742422_1598 (size=31583) 2024-12-05T11:59:06,158 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/2ddb0301117a43fe9e619087cface9c1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2ddb0301117a43fe9e619087cface9c1 2024-12-05T11:59:06,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400006155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:06,163 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/A of 990ce2fe1eb6f27987cd2f0b966a4627 into 2ddb0301117a43fe9e619087cface9c1(size=30.8 K), total size for store is 69.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:06,163 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:06,163 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/A, priority=12, startTime=1733399945568; duration=0sec 2024-12-05T11:59:06,164 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:06,164 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:A 2024-12-05T11:59:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T11:59:06,188 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:59:06,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:59:06,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742423_1599 (size=12151) 2024-12-05T11:59:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-05T11:59:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T11:59:06,191 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:59:06,191 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:59:06,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:59:06,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400006268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:06,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T11:59:06,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T11:59:06,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:06,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:06,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:06,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400006457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:06,461 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:59:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:06,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400006468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:06,469 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_0' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:59:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:06,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400006478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:06,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400006478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:06,483 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:59:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:06,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400006487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:06,491 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:59:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:06,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T11:59:06,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:06,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:06,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:06,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T11:59:06,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/b4b4103641e348e9bf2d6368f1b7cde6 2024-12-05T11:59:06,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/eb0f6e1762a74ef2a7b3045230be2e77 is 50, key is test_row_0/C:col10/1733399945638/Put/seqid=0 2024-12-05T11:59:06,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T11:59:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:06,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742424_1600 (size=12151) 2024-12-05T11:59:06,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400006788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:06,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T11:59:06,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:06,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:06,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:06,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T11:59:06,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T11:59:06,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:06,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:06,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:06,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:07,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/eb0f6e1762a74ef2a7b3045230be2e77 2024-12-05T11:59:07,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/d862bb3eaf4e4046b4048ab9b54c4864 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d862bb3eaf4e4046b4048ab9b54c4864 2024-12-05T11:59:07,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d862bb3eaf4e4046b4048ab9b54c4864, entries=150, sequenceid=249, filesize=30.4 K 2024-12-05T11:59:07,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/b4b4103641e348e9bf2d6368f1b7cde6 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b4b4103641e348e9bf2d6368f1b7cde6 2024-12-05T11:59:07,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b4b4103641e348e9bf2d6368f1b7cde6, entries=150, sequenceid=249, filesize=11.9 K 2024-12-05T11:59:07,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/eb0f6e1762a74ef2a7b3045230be2e77 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/eb0f6e1762a74ef2a7b3045230be2e77 2024-12-05T11:59:07,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/eb0f6e1762a74ef2a7b3045230be2e77, entries=150, sequenceid=249, filesize=11.9 K 2024-12-05T11:59:07,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 990ce2fe1eb6f27987cd2f0b966a4627 in 1106ms, sequenceid=249, compaction requested=true 2024-12-05T11:59:07,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:07,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:59:07,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:07,086 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:59:07,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:59:07,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:59:07,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:59:07,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:59:07,086 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:59:07,087 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:59:07,087 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/A is initiating minor compaction (all files) 2024-12-05T11:59:07,087 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/A in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:07,087 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2ddb0301117a43fe9e619087cface9c1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/bfa4140e5fa24810984912b24f182298, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d862bb3eaf4e4046b4048ab9b54c4864] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=100.0 K 2024-12-05T11:59:07,087 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:07,087 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2ddb0301117a43fe9e619087cface9c1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/bfa4140e5fa24810984912b24f182298, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d862bb3eaf4e4046b4048ab9b54c4864] 2024-12-05T11:59:07,088 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:59:07,088 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/B is initiating minor compaction (all files) 2024-12-05T11:59:07,088 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/B in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:07,088 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/5f5b771bcbbc497aa798d53b895c705b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/e36b1f2730cb4cfcaedca6abdcc581b6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b4b4103641e348e9bf2d6368f1b7cde6] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=36.1 K 2024-12-05T11:59:07,088 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ddb0301117a43fe9e619087cface9c1, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733399944709 2024-12-05T11:59:07,089 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5b771bcbbc497aa798d53b895c705b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733399944709 2024-12-05T11:59:07,089 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfa4140e5fa24810984912b24f182298, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733399945456 2024-12-05T11:59:07,089 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting e36b1f2730cb4cfcaedca6abdcc581b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733399945466 2024-12-05T11:59:07,089 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d862bb3eaf4e4046b4048ab9b54c4864, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733399945638 2024-12-05T11:59:07,089 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting b4b4103641e348e9bf2d6368f1b7cde6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733399945638 2024-12-05T11:59:07,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T11:59:07,111 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:07,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:07,112 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T11:59:07,132 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#B#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:07,133 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/1e5f1ce3b0b64cfab4e42978942c31cf is 50, key is test_row_0/B:col10/1733399945638/Put/seqid=0 2024-12-05T11:59:07,145 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120510696e0d120b478dbc13487a0389e96e_990ce2fe1eb6f27987cd2f0b966a4627 store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:07,147 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120510696e0d120b478dbc13487a0389e96e_990ce2fe1eb6f27987cd2f0b966a4627, store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:07,147 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120510696e0d120b478dbc13487a0389e96e_990ce2fe1eb6f27987cd2f0b966a4627 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:07,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205877643b8926d4519bf7a1e37052d3559_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399946139/Put/seqid=0 2024-12-05T11:59:07,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742425_1601 (size=12731) 2024-12-05T11:59:07,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742426_1602 (size=4469) 2024-12-05T11:59:07,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742427_1603 (size=12454) 2024-12-05T11:59:07,221 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#A#compaction#508 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:07,221 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/ded2569de1774fbbb142fcaa3666f8f9 is 175, key is test_row_0/A:col10/1733399945638/Put/seqid=0 2024-12-05T11:59:07,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742428_1604 (size=31685) 2024-12-05T11:59:07,275 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/ded2569de1774fbbb142fcaa3666f8f9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/ded2569de1774fbbb142fcaa3666f8f9 2024-12-05T11:59:07,283 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/A of 990ce2fe1eb6f27987cd2f0b966a4627 into ded2569de1774fbbb142fcaa3666f8f9(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:07,285 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:07,286 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/A, priority=13, startTime=1733399947086; duration=0sec 2024-12-05T11:59:07,286 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:59:07,286 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:A 2024-12-05T11:59:07,286 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:59:07,287 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:59:07,287 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/C is initiating minor compaction (all files) 2024-12-05T11:59:07,287 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/C in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:07,287 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/1cafb44330104195b2bba9e8e6180a0f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3da0aca2147040a79ed312a3673ff8ff, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/eb0f6e1762a74ef2a7b3045230be2e77] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=36.1 K 2024-12-05T11:59:07,288 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cafb44330104195b2bba9e8e6180a0f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733399944709 2024-12-05T11:59:07,288 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3da0aca2147040a79ed312a3673ff8ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733399945466 2024-12-05T11:59:07,288 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb0f6e1762a74ef2a7b3045230be2e77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733399945638 2024-12-05T11:59:07,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:07,319 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#C#compaction#511 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:07,320 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/7bd9f5f02aae48dd86767b766398fd3e is 50, key is test_row_0/C:col10/1733399945638/Put/seqid=0 2024-12-05T11:59:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T11:59:07,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742429_1605 (size=12731) 2024-12-05T11:59:07,371 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/7bd9f5f02aae48dd86767b766398fd3e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/7bd9f5f02aae48dd86767b766398fd3e 2024-12-05T11:59:07,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400007375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:07,392 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/C of 990ce2fe1eb6f27987cd2f0b966a4627 into 7bd9f5f02aae48dd86767b766398fd3e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:07,392 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:07,392 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/C, priority=13, startTime=1733399947086; duration=0sec 2024-12-05T11:59:07,392 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:07,392 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:C 2024-12-05T11:59:07,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400007488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:07,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:07,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:07,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:07,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:07,624 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/1e5f1ce3b0b64cfab4e42978942c31cf as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/1e5f1ce3b0b64cfab4e42978942c31cf 2024-12-05T11:59:07,625 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205877643b8926d4519bf7a1e37052d3559_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205877643b8926d4519bf7a1e37052d3559_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:07,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/159324bc12c442059e8abc2a62faa9a1, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:07,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/159324bc12c442059e8abc2a62faa9a1 is 175, key is test_row_0/A:col10/1733399946139/Put/seqid=0 2024-12-05T11:59:07,634 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/B of 990ce2fe1eb6f27987cd2f0b966a4627 into 1e5f1ce3b0b64cfab4e42978942c31cf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:07,634 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:07,634 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/B, priority=13, startTime=1733399947086; duration=0sec 2024-12-05T11:59:07,634 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:07,634 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:B 2024-12-05T11:59:07,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742430_1606 (size=31255) 2024-12-05T11:59:07,661 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/159324bc12c442059e8abc2a62faa9a1 2024-12-05T11:59:07,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/78b53a9e30774f7dac71816ced4ef3e0 is 50, key is test_row_0/B:col10/1733399946139/Put/seqid=0 2024-12-05T11:59:07,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400007699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:07,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742431_1607 (size=12301) 2024-12-05T11:59:08,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:08,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400008008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:08,035 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T11:59:08,123 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/78b53a9e30774f7dac71816ced4ef3e0 2024-12-05T11:59:08,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/8e9f78bd87174d798e38158197900f4a is 50, key is test_row_0/C:col10/1733399946139/Put/seqid=0 2024-12-05T11:59:08,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742432_1608 (size=12301) 2024-12-05T11:59:08,163 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/8e9f78bd87174d798e38158197900f4a 2024-12-05T11:59:08,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/159324bc12c442059e8abc2a62faa9a1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/159324bc12c442059e8abc2a62faa9a1 2024-12-05T11:59:08,176 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/159324bc12c442059e8abc2a62faa9a1, entries=150, sequenceid=273, filesize=30.5 K 2024-12-05T11:59:08,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/78b53a9e30774f7dac71816ced4ef3e0 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/78b53a9e30774f7dac71816ced4ef3e0 2024-12-05T11:59:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,191 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/78b53a9e30774f7dac71816ced4ef3e0, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T11:59:08,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/8e9f78bd87174d798e38158197900f4a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/8e9f78bd87174d798e38158197900f4a 2024-12-05T11:59:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,200 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/8e9f78bd87174d798e38158197900f4a, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T11:59:08,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,201 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 990ce2fe1eb6f27987cd2f0b966a4627 in 1089ms, sequenceid=273, compaction requested=false 2024-12-05T11:59:08,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:08,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-05T11:59:08,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-05T11:59:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-05T11:59:08,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0130 sec 2024-12-05T11:59:08,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.0180 sec 2024-12-05T11:59:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T11:59:08,348 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:59:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:59:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-05T11:59:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,351 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:59:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T11:59:08,352 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:59:08,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:59:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T11:59:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-05T11:59:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:08,505 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T11:59:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052ffce32271d24a69afb456a4388e020e_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399947371/Put/seqid=0 2024-12-05T11:59:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742433_1609 (size=9914) 2024-12-05T11:59:08,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:08,597 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052ffce32271d24a69afb456a4388e020e_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052ffce32271d24a69afb456a4388e020e_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:08,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/b3b0445129e049d38fdebc2612d77fe2, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:08,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/b3b0445129e049d38fdebc2612d77fe2 is 175, key is test_row_0/A:col10/1733399947371/Put/seqid=0 2024-12-05T11:59:08,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742434_1610 (size=22561) 2024-12-05T11:59:08,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400008652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T11:59:08,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400008767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:08,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T11:59:08,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400008981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:09,029 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/b3b0445129e049d38fdebc2612d77fe2 2024-12-05T11:59:09,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/4aa785f5123e4872aad6e47b54a254a2 is 50, key is test_row_0/B:col10/1733399947371/Put/seqid=0 2024-12-05T11:59:09,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742435_1611 (size=9857) 2024-12-05T11:59:09,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400009298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T11:59:09,501 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/4aa785f5123e4872aad6e47b54a254a2 2024-12-05T11:59:09,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/652d6342e4c04fed845c43d4a5778969 is 50, key is test_row_0/C:col10/1733399947371/Put/seqid=0 2024-12-05T11:59:09,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742436_1612 (size=9857) 2024-12-05T11:59:09,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400009808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:09,941 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/652d6342e4c04fed845c43d4a5778969 2024-12-05T11:59:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/b3b0445129e049d38fdebc2612d77fe2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/b3b0445129e049d38fdebc2612d77fe2 2024-12-05T11:59:09,959 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/b3b0445129e049d38fdebc2612d77fe2, entries=100, sequenceid=289, filesize=22.0 K 2024-12-05T11:59:09,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/4aa785f5123e4872aad6e47b54a254a2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/4aa785f5123e4872aad6e47b54a254a2 2024-12-05T11:59:09,965 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/4aa785f5123e4872aad6e47b54a254a2, entries=100, sequenceid=289, filesize=9.6 K 2024-12-05T11:59:09,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/652d6342e4c04fed845c43d4a5778969 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/652d6342e4c04fed845c43d4a5778969 2024-12-05T11:59:09,973 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/652d6342e4c04fed845c43d4a5778969, entries=100, sequenceid=289, filesize=9.6 K 2024-12-05T11:59:09,973 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 990ce2fe1eb6f27987cd2f0b966a4627 in 1468ms, sequenceid=289, compaction requested=true 2024-12-05T11:59:09,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:09,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:09,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-05T11:59:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-05T11:59:09,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-05T11:59:09,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6270 sec 2024-12-05T11:59:09,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.6340 sec 2024-12-05T11:59:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:59:10,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T11:59:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T11:59:10,498 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:59:10,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:59:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-05T11:59:10,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f48f36476f5d417c9b3f480639ac4bbb_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399950489/Put/seqid=0 2024-12-05T11:59:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-05T11:59:10,506 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:59:10,507 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:59:10,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:59:10,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400010524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400010524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400010524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400010530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742437_1613 (size=12454) 2024-12-05T11:59:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-05T11:59:10,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400010638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400010638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400010638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400010638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-05T11:59:10,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:10,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:10,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:10,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:10,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:10,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-05T11:59:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:10,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:10,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400010819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,820 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 80666a11a09f,40121,1733399769572 for 'test_row_2' in 990ce2fe1eb6f27987cd2f0b966a4627 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T11:59:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-05T11:59:10,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400010847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400010847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400010847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:10,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400010848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:10,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:10,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:10,945 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:10,949 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f48f36476f5d417c9b3f480639ac4bbb_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f48f36476f5d417c9b3f480639ac4bbb_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:10,951 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/e6f3327cb6de4670a45baba3897a3da1, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:10,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/e6f3327cb6de4670a45baba3897a3da1 is 175, key is test_row_0/A:col10/1733399950489/Put/seqid=0 2024-12-05T11:59:10,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-05T11:59:10,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:10,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:10,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:10,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:10,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:10,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742438_1614 (size=31255) 2024-12-05T11:59:11,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-05T11:59:11,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:11,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:11,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:11,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:11,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-05T11:59:11,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:11,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400011159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400011160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400011160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:11,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400011160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:11,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-05T11:59:11,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:11,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:11,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:11,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:11,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:11,387 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/e6f3327cb6de4670a45baba3897a3da1 2024-12-05T11:59:11,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/483d88abf5c248f49b90c0822b481309 is 50, key is test_row_0/B:col10/1733399950489/Put/seqid=0 2024-12-05T11:59:11,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-05T11:59:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:11,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742439_1615 (size=12301) 2024-12-05T11:59:11,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:11,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:59:11,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/483d88abf5c248f49b90c0822b481309 2024-12-05T11:59:11,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/39263e133f184d66b5d124867e2f7283 is 50, key is test_row_0/C:col10/1733399950489/Put/seqid=0 2024-12-05T11:59:11,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742440_1616 (size=12301) 2024-12-05T11:59:11,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/39263e133f184d66b5d124867e2f7283 2024-12-05T11:59:11,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/e6f3327cb6de4670a45baba3897a3da1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e6f3327cb6de4670a45baba3897a3da1 2024-12-05T11:59:11,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e6f3327cb6de4670a45baba3897a3da1, entries=150, sequenceid=313, filesize=30.5 K 2024-12-05T11:59:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/483d88abf5c248f49b90c0822b481309 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/483d88abf5c248f49b90c0822b481309 2024-12-05T11:59:11,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/483d88abf5c248f49b90c0822b481309, entries=150, sequenceid=313, filesize=12.0 K 2024-12-05T11:59:11,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/39263e133f184d66b5d124867e2f7283 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/39263e133f184d66b5d124867e2f7283 2024-12-05T11:59:11,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/39263e133f184d66b5d124867e2f7283, entries=150, sequenceid=313, filesize=12.0 K 2024-12-05T11:59:11,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 990ce2fe1eb6f27987cd2f0b966a4627 in 1037ms, sequenceid=313, compaction requested=true 2024-12-05T11:59:11,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:11,528 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:59:11,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:59:11,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:11,529 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 116756 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:59:11,529 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/A is initiating minor compaction (all files) 2024-12-05T11:59:11,529 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/A in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:11,529 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/ded2569de1774fbbb142fcaa3666f8f9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/159324bc12c442059e8abc2a62faa9a1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/b3b0445129e049d38fdebc2612d77fe2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e6f3327cb6de4670a45baba3897a3da1] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=114.0 K 2024-12-05T11:59:11,529 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:11,529 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. files: [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/ded2569de1774fbbb142fcaa3666f8f9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/159324bc12c442059e8abc2a62faa9a1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/b3b0445129e049d38fdebc2612d77fe2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e6f3327cb6de4670a45baba3897a3da1] 2024-12-05T11:59:11,530 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ded2569de1774fbbb142fcaa3666f8f9, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733399945638 2024-12-05T11:59:11,530 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 159324bc12c442059e8abc2a62faa9a1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733399946131 2024-12-05T11:59:11,531 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:59:11,532 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3b0445129e049d38fdebc2612d77fe2, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733399947348 2024-12-05T11:59:11,533 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f3327cb6de4670a45baba3897a3da1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733399948642 2024-12-05T11:59:11,535 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47190 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:59:11,535 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/B is initiating minor compaction (all files) 2024-12-05T11:59:11,535 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/B in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:11,535 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/1e5f1ce3b0b64cfab4e42978942c31cf, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/78b53a9e30774f7dac71816ced4ef3e0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/4aa785f5123e4872aad6e47b54a254a2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/483d88abf5c248f49b90c0822b481309] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=46.1 K 2024-12-05T11:59:11,536 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e5f1ce3b0b64cfab4e42978942c31cf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733399945638 2024-12-05T11:59:11,537 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 78b53a9e30774f7dac71816ced4ef3e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733399946131 2024-12-05T11:59:11,537 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aa785f5123e4872aad6e47b54a254a2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733399947348 2024-12-05T11:59:11,537 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 483d88abf5c248f49b90c0822b481309, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733399948642 2024-12-05T11:59:11,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:59:11,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:11,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:59:11,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:59:11,551 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:11,555 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#B#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:11,556 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/aecd18fd3dd946e8bdc1fdd3df017f4a is 50, key is test_row_0/B:col10/1733399950489/Put/seqid=0 2024-12-05T11:59:11,557 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120515e1cb9c858a4960a5fa5e1aeb1bf2fd_990ce2fe1eb6f27987cd2f0b966a4627 store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:11,559 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120515e1cb9c858a4960a5fa5e1aeb1bf2fd_990ce2fe1eb6f27987cd2f0b966a4627, store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:11,559 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120515e1cb9c858a4960a5fa5e1aeb1bf2fd_990ce2fe1eb6f27987cd2f0b966a4627 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:11,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742441_1617 (size=13017) 2024-12-05T11:59:11,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-05T11:59:11,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:11,594 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T11:59:11,599 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/aecd18fd3dd946e8bdc1fdd3df017f4a as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/aecd18fd3dd946e8bdc1fdd3df017f4a 2024-12-05T11:59:11,604 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/B of 990ce2fe1eb6f27987cd2f0b966a4627 into aecd18fd3dd946e8bdc1fdd3df017f4a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:11,604 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:11,604 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/B, priority=12, startTime=1733399951531; duration=0sec 2024-12-05T11:59:11,604 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:59:11,604 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:B 2024-12-05T11:59:11,604 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:59:11,606 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47190 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:59:11,606 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1541): 990ce2fe1eb6f27987cd2f0b966a4627/C is initiating minor compaction (all files) 2024-12-05T11:59:11,606 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 990ce2fe1eb6f27987cd2f0b966a4627/C in TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:11,606 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/7bd9f5f02aae48dd86767b766398fd3e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/8e9f78bd87174d798e38158197900f4a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/652d6342e4c04fed845c43d4a5778969, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/39263e133f184d66b5d124867e2f7283] into tmpdir=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp, totalSize=46.1 K 2024-12-05T11:59:11,606 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd9f5f02aae48dd86767b766398fd3e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733399945638 2024-12-05T11:59:11,607 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e9f78bd87174d798e38158197900f4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733399946131 2024-12-05T11:59:11,607 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 652d6342e4c04fed845c43d4a5778969, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733399947348 2024-12-05T11:59:11,607 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] compactions.Compactor(225): Compacting 39263e133f184d66b5d124867e2f7283, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733399948642 2024-12-05T11:59:11,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742442_1618 (size=4469) 2024-12-05T11:59:11,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e653ecc402fe416fbe2ce5c4178a9ff9_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399950524/Put/seqid=0 2024-12-05T11:59:11,623 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#C#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:11,624 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/9cec692a2ebf4c3390846a7b4807a0a4 is 50, key is test_row_0/C:col10/1733399950489/Put/seqid=0 2024-12-05T11:59:11,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742443_1619 (size=12454) 2024-12-05T11:59:11,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:11,639 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e653ecc402fe416fbe2ce5c4178a9ff9_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e653ecc402fe416fbe2ce5c4178a9ff9_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:11,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/0f499cec035b49cba54d0b83a030e3bd, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:11,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/0f499cec035b49cba54d0b83a030e3bd is 175, key is test_row_0/A:col10/1733399950524/Put/seqid=0 2024-12-05T11:59:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-05T11:59:11,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742444_1620 (size=13017) 2024-12-05T11:59:11,663 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/9cec692a2ebf4c3390846a7b4807a0a4 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/9cec692a2ebf4c3390846a7b4807a0a4 2024-12-05T11:59:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:11,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:11,671 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/C of 990ce2fe1eb6f27987cd2f0b966a4627 into 9cec692a2ebf4c3390846a7b4807a0a4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:11,671 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:11,671 INFO [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/C, priority=12, startTime=1733399951541; duration=0sec 2024-12-05T11:59:11,671 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:11,671 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:C 2024-12-05T11:59:11,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742445_1621 (size=31255) 2024-12-05T11:59:11,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400011703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:11,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400011703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:11,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400011704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:11,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:11,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:11,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400011708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:11,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400011820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400011820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:11,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400011820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:11,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400011821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,010 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 990ce2fe1eb6f27987cd2f0b966a4627#A#compaction#520 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-05T11:59:12,010 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/056d1733be074290bc5cba45dcc6cee7 is 175, key is test_row_0/A:col10/1733399950489/Put/seqid=0 2024-12-05T11:59:12,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400012028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400012028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400012029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400012029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742446_1622 (size=31971) 2024-12-05T11:59:12,075 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/0f499cec035b49cba54d0b83a030e3bd 2024-12-05T11:59:12,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/b60baf65f9594815bfa2ada61e60531c is 50, key is test_row_0/B:col10/1733399950524/Put/seqid=0 2024-12-05T11:59:12,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742447_1623 (size=12301) 2024-12-05T11:59:12,126 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/b60baf65f9594815bfa2ada61e60531c 2024-12-05T11:59:12,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/27295fe7f0e145fa9d025d39200cd5bf is 50, key is test_row_0/C:col10/1733399950524/Put/seqid=0 2024-12-05T11:59:12,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742448_1624 (size=12301) 2024-12-05T11:59:12,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400012337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400012337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400012337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400012338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,451 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/056d1733be074290bc5cba45dcc6cee7 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/056d1733be074290bc5cba45dcc6cee7 2024-12-05T11:59:12,456 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 990ce2fe1eb6f27987cd2f0b966a4627/A of 990ce2fe1eb6f27987cd2f0b966a4627 into 056d1733be074290bc5cba45dcc6cee7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:59:12,456 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:12,456 INFO [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., storeName=990ce2fe1eb6f27987cd2f0b966a4627/A, priority=12, startTime=1733399951528; duration=0sec 2024-12-05T11:59:12,456 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:12,456 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:A 2024-12-05T11:59:12,574 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/27295fe7f0e145fa9d025d39200cd5bf 2024-12-05T11:59:12,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/0f499cec035b49cba54d0b83a030e3bd as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0f499cec035b49cba54d0b83a030e3bd 2024-12-05T11:59:12,585 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0f499cec035b49cba54d0b83a030e3bd, entries=150, sequenceid=325, filesize=30.5 K 2024-12-05T11:59:12,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/b60baf65f9594815bfa2ada61e60531c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b60baf65f9594815bfa2ada61e60531c 2024-12-05T11:59:12,592 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b60baf65f9594815bfa2ada61e60531c, entries=150, sequenceid=325, filesize=12.0 K 2024-12-05T11:59:12,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/27295fe7f0e145fa9d025d39200cd5bf as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/27295fe7f0e145fa9d025d39200cd5bf 2024-12-05T11:59:12,598 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/27295fe7f0e145fa9d025d39200cd5bf, entries=150, sequenceid=325, filesize=12.0 K 2024-12-05T11:59:12,600 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 990ce2fe1eb6f27987cd2f0b966a4627 in 1006ms, sequenceid=325, compaction requested=false 2024-12-05T11:59:12,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:12,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:12,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-05T11:59:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-05T11:59:12,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-05T11:59:12,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0940 sec 2024-12-05T11:59:12,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.1030 sec 2024-12-05T11:59:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-05T11:59:12,658 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:59:12,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:59:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-05T11:59:12,661 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:59:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-05T11:59:12,662 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:59:12,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:59:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-05T11:59:12,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-05T11:59:12,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:12,817 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T11:59:12,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:12,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ac9913ef8a82443e87c721e57c8a2d4a_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399951697/Put/seqid=0 2024-12-05T11:59:12,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742449_1625 (size=12454) 2024-12-05T11:59:12,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:12,876 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ac9913ef8a82443e87c721e57c8a2d4a_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ac9913ef8a82443e87c721e57c8a2d4a_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:12,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/2050500d7ff440e78e18e1257f90a3b9, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:12,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/2050500d7ff440e78e18e1257f90a3b9 is 175, key is test_row_0/A:col10/1733399951697/Put/seqid=0 2024-12-05T11:59:12,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400012851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400012853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400012884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400012884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400012891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742450_1626 (size=31255) 2024-12-05T11:59:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-05T11:59:12,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400012988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400012988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:12,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400012998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400012998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400012998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400013201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400013201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:13,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400013208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:13,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400013209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400013209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-05T11:59:13,319 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/2050500d7ff440e78e18e1257f90a3b9 2024-12-05T11:59:13,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/503c9ee2f4c84ac98d7f75825d9f736d is 50, key is test_row_0/B:col10/1733399951697/Put/seqid=0 2024-12-05T11:59:13,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742451_1627 (size=12301) 2024-12-05T11:59:13,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400013508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400013508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400013518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400013518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400013518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:13,794 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/503c9ee2f4c84ac98d7f75825d9f736d 2024-12-05T11:59:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-05T11:59:13,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/1422aaf052e64eb5bccede919d5a6eb2 is 50, key is test_row_0/C:col10/1733399951697/Put/seqid=0 2024-12-05T11:59:13,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742452_1628 (size=12301) 2024-12-05T11:59:14,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400014018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:14,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400014018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:14,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400014028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:14,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400014028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:14,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400014028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:14,237 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/1422aaf052e64eb5bccede919d5a6eb2 2024-12-05T11:59:14,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/2050500d7ff440e78e18e1257f90a3b9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2050500d7ff440e78e18e1257f90a3b9 2024-12-05T11:59:14,246 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2050500d7ff440e78e18e1257f90a3b9, entries=150, sequenceid=352, filesize=30.5 K 2024-12-05T11:59:14,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/503c9ee2f4c84ac98d7f75825d9f736d as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/503c9ee2f4c84ac98d7f75825d9f736d 2024-12-05T11:59:14,258 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/503c9ee2f4c84ac98d7f75825d9f736d, entries=150, sequenceid=352, filesize=12.0 K 2024-12-05T11:59:14,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/1422aaf052e64eb5bccede919d5a6eb2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/1422aaf052e64eb5bccede919d5a6eb2 2024-12-05T11:59:14,264 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/1422aaf052e64eb5bccede919d5a6eb2, entries=150, sequenceid=352, filesize=12.0 K 2024-12-05T11:59:14,265 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for 990ce2fe1eb6f27987cd2f0b966a4627 in 1448ms, sequenceid=352, compaction requested=true 2024-12-05T11:59:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-05T11:59:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-05T11:59:14,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-05T11:59:14,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6050 sec 2024-12-05T11:59:14,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.6130 sec 2024-12-05T11:59:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-05T11:59:14,808 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:59:14,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:59:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-12-05T11:59:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-05T11:59:14,811 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:59:14,811 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:59:14,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:59:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-05T11:59:14,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-05T11:59:14,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:14,964 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T11:59:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056108337036c74ee9835192e35ec1c1ec_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399952873/Put/seqid=0 2024-12-05T11:59:15,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742453_1629 (size=12454) 2024-12-05T11:59:15,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:15,016 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056108337036c74ee9835192e35ec1c1ec_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056108337036c74ee9835192e35ec1c1ec_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:15,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/0d7ac4a441404321b3cab5019c578298, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:15,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/0d7ac4a441404321b3cab5019c578298 is 175, key is test_row_0/A:col10/1733399952873/Put/seqid=0 2024-12-05T11:59:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. as already flushing 2024-12-05T11:59:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:15,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742454_1630 (size=31255) 2024-12-05T11:59:15,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400015104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400015113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400015114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400015115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400015115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-05T11:59:15,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400015217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400015228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400015228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400015228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400015229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400015427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-05T11:59:15,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400015438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400015438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400015438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400015438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,450 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/0d7ac4a441404321b3cab5019c578298 2024-12-05T11:59:15,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/a0eca54303cc47ff9d15bfa5f2c3b9bb is 50, key is test_row_0/B:col10/1733399952873/Put/seqid=0 2024-12-05T11:59:15,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742455_1631 (size=12301) 2024-12-05T11:59:15,585 INFO [Thread-2389 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2389. 2024-12-05T11:59:15,585 DEBUG [Thread-2389 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:59:15,586 DEBUG [Thread-2389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:15,586 DEBUG [Thread-2389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:15,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:59:15,587 INFO [Thread-2393 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2393. 2024-12-05T11:59:15,587 DEBUG [Thread-2393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:59:15,587 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:15,587 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:15,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:59:15,588 INFO [Thread-2391 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2391. 2024-12-05T11:59:15,588 DEBUG [Thread-2391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:59:15,588 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:15,588 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:15,588 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:59:15,591 INFO [Thread-2395 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2395. 2024-12-05T11:59:15,591 DEBUG [Thread-2395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:59:15,591 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:15,591 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:15,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:59:15,591 INFO [Thread-2397 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2397. 2024-12-05T11:59:15,591 DEBUG [Thread-2397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:59:15,592 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:15,592 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:15,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:59:15,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44502 deadline: 1733400015737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44420 deadline: 1733400015748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44484 deadline: 1733400015748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44496 deadline: 1733400015748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T11:59:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44456 deadline: 1733400015748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 2024-12-05T11:59:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 , the old value is region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:59:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., hostname=80666a11a09f,40121,1733399769572, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:59:15,872 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/a0eca54303cc47ff9d15bfa5f2c3b9bb 2024-12-05T11:59:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/a94fb3e709454359a1c87e2958486b0c is 50, key is test_row_0/C:col10/1733399952873/Put/seqid=0 2024-12-05T11:59:15,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742456_1632 (size=12301) 2024-12-05T11:59:15,881 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/a94fb3e709454359a1c87e2958486b0c 2024-12-05T11:59:15,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/0d7ac4a441404321b3cab5019c578298 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0d7ac4a441404321b3cab5019c578298 2024-12-05T11:59:15,900 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0d7ac4a441404321b3cab5019c578298, entries=150, sequenceid=366, filesize=30.5 K 2024-12-05T11:59:15,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/a0eca54303cc47ff9d15bfa5f2c3b9bb as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/a0eca54303cc47ff9d15bfa5f2c3b9bb 2024-12-05T11:59:15,906 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/a0eca54303cc47ff9d15bfa5f2c3b9bb, entries=150, sequenceid=366, filesize=12.0 K 2024-12-05T11:59:15,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/a94fb3e709454359a1c87e2958486b0c as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/a94fb3e709454359a1c87e2958486b0c 2024-12-05T11:59:15,910 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/a94fb3e709454359a1c87e2958486b0c, entries=150, sequenceid=366, filesize=12.0 K 2024-12-05T11:59:15,911 INFO [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for 990ce2fe1eb6f27987cd2f0b966a4627 in 948ms, sequenceid=366, compaction requested=true 2024-12-05T11:59:15,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:15,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:15,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-12-05T11:59:15,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-12-05T11:59:15,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-05T11:59:15,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1010 sec 2024-12-05T11:59:15,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.1070 sec 2024-12-05T11:59:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-05T11:59:15,948 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:59:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40121 {}] regionserver.HRegion(8855): Flush requested on 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:16,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T11:59:16,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T11:59:16,249 INFO [Thread-2382 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2382. 2024-12-05T11:59:16,249 DEBUG [Thread-2382 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:59:16,249 DEBUG [Thread-2382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:16,249 DEBUG [Thread-2382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:16,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:59:16,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a0f3970cdda94deda0e195f80cd51c7f_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399955110/Put/seqid=0 2024-12-05T11:59:16,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742457_1633 (size=12454) 2024-12-05T11:59:16,260 INFO [Thread-2378 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2378. 2024-12-05T11:59:16,260 DEBUG [Thread-2378 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:59:16,260 INFO [Thread-2386 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2386. 2024-12-05T11:59:16,260 DEBUG [Thread-2378 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:16,260 DEBUG [Thread-2386 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:59:16,260 DEBUG [Thread-2386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:16,260 DEBUG [Thread-2378 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:16,260 DEBUG [Thread-2386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:16,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:59:16,261 INFO [Thread-2380 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2380. 2024-12-05T11:59:16,261 DEBUG [Thread-2380 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:59:16,261 DEBUG [Thread-2380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:16,261 DEBUG [Thread-2380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:16,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:59:16,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:59:16,267 INFO [Thread-2384 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2384. 2024-12-05T11:59:16,267 DEBUG [Thread-2384 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:59:16,267 DEBUG [Thread-2384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:16,267 DEBUG [Thread-2384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:16,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:59:16,268 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 173 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3583 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3556 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3552 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3562 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3513 2024-12-05T11:59:16,268 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T11:59:16,268 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:59:16,268 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:59:16,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:16,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:16,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:59:16,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T11:59:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T11:59:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-05T11:59:16,273 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399956272"}]},"ts":"1733399956272"} 2024-12-05T11:59:16,274 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T11:59:16,274 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T11:59:16,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:59:16,275 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a0f3970cdda94deda0e195f80cd51c7f_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a0f3970cdda94deda0e195f80cd51c7f_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:16,276 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/655f8fcbc9014eadb2a993040b06c8d7, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:16,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/655f8fcbc9014eadb2a993040b06c8d7 is 175, key is test_row_0/A:col10/1733399955110/Put/seqid=0 2024-12-05T11:59:16,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, UNASSIGN}] 2024-12-05T11:59:16,278 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, UNASSIGN 2024-12-05T11:59:16,278 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=990ce2fe1eb6f27987cd2f0b966a4627, regionState=CLOSING, regionLocation=80666a11a09f,40121,1733399769572 2024-12-05T11:59:16,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, UNASSIGN because future has completed 2024-12-05T11:59:16,281 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:59:16,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572}] 2024-12-05T11:59:16,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742458_1634 (size=31255) 2024-12-05T11:59:16,286 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/655f8fcbc9014eadb2a993040b06c8d7 2024-12-05T11:59:16,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/e3038c7c56aa4c28bac33110655a8550 is 50, key is test_row_0/B:col10/1733399955110/Put/seqid=0 2024-12-05T11:59:16,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742459_1635 (size=12301) 2024-12-05T11:59:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-05T11:59:16,435 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:16,435 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:59:16,435 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing 990ce2fe1eb6f27987cd2f0b966a4627, disabling compactions & flushes 2024-12-05T11:59:16,435 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-05T11:59:16,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/e3038c7c56aa4c28bac33110655a8550 2024-12-05T11:59:16,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/3c52969b0d7d4d35b5be1ab3315783e3 is 50, key is test_row_0/C:col10/1733399955110/Put/seqid=0 2024-12-05T11:59:16,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742460_1636 (size=12301) 2024-12-05T11:59:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-05T11:59:17,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/3c52969b0d7d4d35b5be1ab3315783e3 2024-12-05T11:59:17,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/655f8fcbc9014eadb2a993040b06c8d7 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/655f8fcbc9014eadb2a993040b06c8d7 2024-12-05T11:59:17,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/655f8fcbc9014eadb2a993040b06c8d7, entries=150, sequenceid=391, filesize=30.5 K 2024-12-05T11:59:17,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/e3038c7c56aa4c28bac33110655a8550 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/e3038c7c56aa4c28bac33110655a8550 2024-12-05T11:59:17,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/e3038c7c56aa4c28bac33110655a8550, entries=150, sequenceid=391, filesize=12.0 K 2024-12-05T11:59:17,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/3c52969b0d7d4d35b5be1ab3315783e3 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3c52969b0d7d4d35b5be1ab3315783e3 2024-12-05T11:59:17,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3c52969b0d7d4d35b5be1ab3315783e3, entries=150, sequenceid=391, filesize=12.0 K 2024-12-05T11:59:17,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for 990ce2fe1eb6f27987cd2f0b966a4627 in 868ms, sequenceid=391, compaction requested=true 2024-12-05T11:59:17,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 990ce2fe1eb6f27987cd2f0b966a4627: 2024-12-05T11:59:17,117 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:17,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:59:17,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:17,117 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. because compaction request was cancelled 2024-12-05T11:59:17,117 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:17,117 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:A 2024-12-05T11:59:17,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:59:17,117 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. after waiting 0 ms 2024-12-05T11:59:17,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:17,117 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. because compaction request was cancelled 2024-12-05T11:59:17,117 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:17,117 DEBUG [RS:0;80666a11a09f:40121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:B 2024-12-05T11:59:17,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 990ce2fe1eb6f27987cd2f0b966a4627:C, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:59:17,117 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. because compaction request was cancelled 2024-12-05T11:59:17,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:59:17,117 DEBUG [RS:0;80666a11a09f:40121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 990ce2fe1eb6f27987cd2f0b966a4627:C 2024-12-05T11:59:17,117 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing 990ce2fe1eb6f27987cd2f0b966a4627 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-05T11:59:17,122 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120536ae734c68934827a079ca50b4ebbd87_990ce2fe1eb6f27987cd2f0b966a4627 is 50, key is test_row_0/A:col10/1733399956260/Put/seqid=0 2024-12-05T11:59:17,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742461_1637 (size=12454) 2024-12-05T11:59:17,126 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:59:17,128 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120536ae734c68934827a079ca50b4ebbd87_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120536ae734c68934827a079ca50b4ebbd87_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:17,129 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/2e2b47daafc44442a4b2e94cce9d2fe1, store: [table=TestAcidGuarantees family=A region=990ce2fe1eb6f27987cd2f0b966a4627] 2024-12-05T11:59:17,129 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/2e2b47daafc44442a4b2e94cce9d2fe1 is 175, key is test_row_0/A:col10/1733399956260/Put/seqid=0 2024-12-05T11:59:17,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742462_1638 (size=31255) 2024-12-05T11:59:17,133 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/2e2b47daafc44442a4b2e94cce9d2fe1 2024-12-05T11:59:17,137 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/d9b5f13dbe8746cd9b95362f06b77bd2 is 50, key is test_row_0/B:col10/1733399956260/Put/seqid=0 2024-12-05T11:59:17,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742463_1639 (size=12301) 2024-12-05T11:59:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-05T11:59:17,542 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/d9b5f13dbe8746cd9b95362f06b77bd2 2024-12-05T11:59:17,548 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/e670852710da4486b926afd7f20a04ca is 50, key is test_row_0/C:col10/1733399956260/Put/seqid=0 2024-12-05T11:59:17,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742464_1640 (size=12301) 2024-12-05T11:59:17,579 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/e670852710da4486b926afd7f20a04ca 2024-12-05T11:59:17,585 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/A/2e2b47daafc44442a4b2e94cce9d2fe1 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2e2b47daafc44442a4b2e94cce9d2fe1 2024-12-05T11:59:17,590 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2e2b47daafc44442a4b2e94cce9d2fe1, entries=150, sequenceid=398, filesize=30.5 K 2024-12-05T11:59:17,590 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/B/d9b5f13dbe8746cd9b95362f06b77bd2 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/d9b5f13dbe8746cd9b95362f06b77bd2 2024-12-05T11:59:17,596 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/d9b5f13dbe8746cd9b95362f06b77bd2, entries=150, sequenceid=398, filesize=12.0 K 2024-12-05T11:59:17,597 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/.tmp/C/e670852710da4486b926afd7f20a04ca as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/e670852710da4486b926afd7f20a04ca 2024-12-05T11:59:17,602 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/e670852710da4486b926afd7f20a04ca, entries=150, sequenceid=398, filesize=12.0 K 2024-12-05T11:59:17,603 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 990ce2fe1eb6f27987cd2f0b966a4627 in 486ms, sequenceid=398, compaction requested=true 2024-12-05T11:59:17,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e23ad39c4a624da0af19dde105c5a21d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8bbdf99191a840abb75c1186814dd657, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/78deb9bb1041463f85c4fff391429cba, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8ef0ef2530c14eca9d54d31dba0b84ab, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/30e1ee2cd5d44283a41af23da498295d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/781552fdd0e7414499dc0b71f9b0fb96, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c3498e67baf94e7293684d6d67371d6f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/5c4e8bd2180b414f8e23de3de6e24a66, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/02daf463d1034653baab0652719de198, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0ec2d28892754cc6b7a4b256db2257bc, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d7b786f664b64a159e713e8af6d73493, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/460ca6d138684386b1db8952c25857a2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/abb34a191ea04db3bc3af7cb18de9180, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c623ae9f84734466835d219ae39dd277, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2ddb0301117a43fe9e619087cface9c1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/bfa4140e5fa24810984912b24f182298, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/ded2569de1774fbbb142fcaa3666f8f9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d862bb3eaf4e4046b4048ab9b54c4864, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/159324bc12c442059e8abc2a62faa9a1, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/b3b0445129e049d38fdebc2612d77fe2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e6f3327cb6de4670a45baba3897a3da1] to archive 2024-12-05T11:59:17,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:59:17,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e23ad39c4a624da0af19dde105c5a21d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e23ad39c4a624da0af19dde105c5a21d 2024-12-05T11:59:17,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8bbdf99191a840abb75c1186814dd657 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8bbdf99191a840abb75c1186814dd657 2024-12-05T11:59:17,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/78deb9bb1041463f85c4fff391429cba to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/78deb9bb1041463f85c4fff391429cba 2024-12-05T11:59:17,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8ef0ef2530c14eca9d54d31dba0b84ab to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/8ef0ef2530c14eca9d54d31dba0b84ab 2024-12-05T11:59:17,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/30e1ee2cd5d44283a41af23da498295d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/30e1ee2cd5d44283a41af23da498295d 2024-12-05T11:59:17,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/781552fdd0e7414499dc0b71f9b0fb96 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/781552fdd0e7414499dc0b71f9b0fb96 2024-12-05T11:59:17,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c3498e67baf94e7293684d6d67371d6f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c3498e67baf94e7293684d6d67371d6f 2024-12-05T11:59:17,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/5c4e8bd2180b414f8e23de3de6e24a66 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/5c4e8bd2180b414f8e23de3de6e24a66 2024-12-05T11:59:17,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/02daf463d1034653baab0652719de198 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/02daf463d1034653baab0652719de198 2024-12-05T11:59:17,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0ec2d28892754cc6b7a4b256db2257bc to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0ec2d28892754cc6b7a4b256db2257bc 2024-12-05T11:59:17,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d7b786f664b64a159e713e8af6d73493 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d7b786f664b64a159e713e8af6d73493 2024-12-05T11:59:17,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/460ca6d138684386b1db8952c25857a2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/460ca6d138684386b1db8952c25857a2 2024-12-05T11:59:17,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/abb34a191ea04db3bc3af7cb18de9180 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/abb34a191ea04db3bc3af7cb18de9180 2024-12-05T11:59:17,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c623ae9f84734466835d219ae39dd277 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/c623ae9f84734466835d219ae39dd277 2024-12-05T11:59:17,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2ddb0301117a43fe9e619087cface9c1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2ddb0301117a43fe9e619087cface9c1 2024-12-05T11:59:17,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/bfa4140e5fa24810984912b24f182298 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/bfa4140e5fa24810984912b24f182298 2024-12-05T11:59:17,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/ded2569de1774fbbb142fcaa3666f8f9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/ded2569de1774fbbb142fcaa3666f8f9 2024-12-05T11:59:17,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d862bb3eaf4e4046b4048ab9b54c4864 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/d862bb3eaf4e4046b4048ab9b54c4864 2024-12-05T11:59:17,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/159324bc12c442059e8abc2a62faa9a1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/159324bc12c442059e8abc2a62faa9a1 2024-12-05T11:59:17,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/b3b0445129e049d38fdebc2612d77fe2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/b3b0445129e049d38fdebc2612d77fe2 2024-12-05T11:59:17,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e6f3327cb6de4670a45baba3897a3da1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/e6f3327cb6de4670a45baba3897a3da1 2024-12-05T11:59:17,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/2d624c0f22dd4683833eb074e4f3403b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/09e6b800ce4f4ba793e6b9df7926a25c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/14128a2eb3ad4d81b15d7e79d345593b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/d3fde9e9774b4b55ab2ffee7833a9634, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b7ee54576da644d3bd8c85ec47744293, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/db788f00c3204176adf8f96db4d832ad, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/24e90f1561ff459b829d9e21371b8abb, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/ac8e082d4ebe4856a956605b8c6b8f26, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/9ccebcb980884c0795d7923d9731b595, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/2744cfdd9e1c40cb808cd5e1bc7c61a4, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/8a37bf95bba44cb5b6406a632b35fa7d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/f1c7f3b3e47b4c4184b62aaf9b86a76c, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/ec2b5c3898e84137ba3ab47ecea3f6d0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/5f5b771bcbbc497aa798d53b895c705b, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/25d306dd2c4e4b11aea83b7f5dec0dc0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/e36b1f2730cb4cfcaedca6abdcc581b6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/1e5f1ce3b0b64cfab4e42978942c31cf, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b4b4103641e348e9bf2d6368f1b7cde6, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/78b53a9e30774f7dac71816ced4ef3e0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/4aa785f5123e4872aad6e47b54a254a2, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/483d88abf5c248f49b90c0822b481309] to archive 2024-12-05T11:59:17,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:59:17,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/2d624c0f22dd4683833eb074e4f3403b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/2d624c0f22dd4683833eb074e4f3403b 2024-12-05T11:59:17,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/09e6b800ce4f4ba793e6b9df7926a25c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/09e6b800ce4f4ba793e6b9df7926a25c 2024-12-05T11:59:17,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/14128a2eb3ad4d81b15d7e79d345593b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/14128a2eb3ad4d81b15d7e79d345593b 2024-12-05T11:59:17,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/d3fde9e9774b4b55ab2ffee7833a9634 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/d3fde9e9774b4b55ab2ffee7833a9634 2024-12-05T11:59:17,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b7ee54576da644d3bd8c85ec47744293 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b7ee54576da644d3bd8c85ec47744293 2024-12-05T11:59:17,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/db788f00c3204176adf8f96db4d832ad to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/db788f00c3204176adf8f96db4d832ad 2024-12-05T11:59:17,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/24e90f1561ff459b829d9e21371b8abb to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/24e90f1561ff459b829d9e21371b8abb 2024-12-05T11:59:17,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/ac8e082d4ebe4856a956605b8c6b8f26 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/ac8e082d4ebe4856a956605b8c6b8f26 2024-12-05T11:59:17,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/9ccebcb980884c0795d7923d9731b595 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/9ccebcb980884c0795d7923d9731b595 2024-12-05T11:59:17,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/2744cfdd9e1c40cb808cd5e1bc7c61a4 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/2744cfdd9e1c40cb808cd5e1bc7c61a4 2024-12-05T11:59:17,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/8a37bf95bba44cb5b6406a632b35fa7d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/8a37bf95bba44cb5b6406a632b35fa7d 2024-12-05T11:59:17,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/f1c7f3b3e47b4c4184b62aaf9b86a76c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/f1c7f3b3e47b4c4184b62aaf9b86a76c 2024-12-05T11:59:17,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/ec2b5c3898e84137ba3ab47ecea3f6d0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/ec2b5c3898e84137ba3ab47ecea3f6d0 2024-12-05T11:59:17,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/5f5b771bcbbc497aa798d53b895c705b to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/5f5b771bcbbc497aa798d53b895c705b 2024-12-05T11:59:17,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/25d306dd2c4e4b11aea83b7f5dec0dc0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/25d306dd2c4e4b11aea83b7f5dec0dc0 2024-12-05T11:59:17,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/e36b1f2730cb4cfcaedca6abdcc581b6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/e36b1f2730cb4cfcaedca6abdcc581b6 2024-12-05T11:59:17,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/1e5f1ce3b0b64cfab4e42978942c31cf to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/1e5f1ce3b0b64cfab4e42978942c31cf 2024-12-05T11:59:17,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b4b4103641e348e9bf2d6368f1b7cde6 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b4b4103641e348e9bf2d6368f1b7cde6 2024-12-05T11:59:17,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/78b53a9e30774f7dac71816ced4ef3e0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/78b53a9e30774f7dac71816ced4ef3e0 2024-12-05T11:59:17,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/4aa785f5123e4872aad6e47b54a254a2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/4aa785f5123e4872aad6e47b54a254a2 2024-12-05T11:59:17,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/483d88abf5c248f49b90c0822b481309 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/483d88abf5c248f49b90c0822b481309 2024-12-05T11:59:17,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/2409820fa7f045e1a46ea35c77075949, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/6d3fbf92042643b8a08566dcca4dfc91, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/dd33f64d81d041dbbefa05b6da4b60f0, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/e26b4838f21c4417a589010d3877c60d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/b802f4b28b054183994f3c34ec845c84, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/c2f11cd7907a469fa3998b03832ffa9e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/650412bf4b3c4aaf9da84352c142d1d9, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/cb3894d0de4d4b6e89f827fa6805c0cc, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/bbd06002f4ff41cab1c38a4f9f317f88, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3aee42c567954d96ad4e3703ceb6472e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/9df967d083a8470f98c255932b2c6524, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/36ee47ed0fa84536a89c53154811270d, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/ae25df7e441a436aa57f96a53a247e45, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/1cafb44330104195b2bba9e8e6180a0f, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/d07520c664fb4e36a1ae0e4a5630ce97, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3da0aca2147040a79ed312a3673ff8ff, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/7bd9f5f02aae48dd86767b766398fd3e, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/eb0f6e1762a74ef2a7b3045230be2e77, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/8e9f78bd87174d798e38158197900f4a, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/652d6342e4c04fed845c43d4a5778969, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/39263e133f184d66b5d124867e2f7283] to archive 2024-12-05T11:59:17,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:59:17,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/2409820fa7f045e1a46ea35c77075949 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/2409820fa7f045e1a46ea35c77075949 2024-12-05T11:59:17,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/6d3fbf92042643b8a08566dcca4dfc91 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/6d3fbf92042643b8a08566dcca4dfc91 2024-12-05T11:59:17,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/dd33f64d81d041dbbefa05b6da4b60f0 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/dd33f64d81d041dbbefa05b6da4b60f0 2024-12-05T11:59:17,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/e26b4838f21c4417a589010d3877c60d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/e26b4838f21c4417a589010d3877c60d 2024-12-05T11:59:17,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/b802f4b28b054183994f3c34ec845c84 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/b802f4b28b054183994f3c34ec845c84 2024-12-05T11:59:17,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/c2f11cd7907a469fa3998b03832ffa9e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/c2f11cd7907a469fa3998b03832ffa9e 2024-12-05T11:59:17,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/650412bf4b3c4aaf9da84352c142d1d9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/650412bf4b3c4aaf9da84352c142d1d9 2024-12-05T11:59:17,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/cb3894d0de4d4b6e89f827fa6805c0cc to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/cb3894d0de4d4b6e89f827fa6805c0cc 2024-12-05T11:59:17,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/bbd06002f4ff41cab1c38a4f9f317f88 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/bbd06002f4ff41cab1c38a4f9f317f88 2024-12-05T11:59:17,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3aee42c567954d96ad4e3703ceb6472e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3aee42c567954d96ad4e3703ceb6472e 2024-12-05T11:59:17,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/9df967d083a8470f98c255932b2c6524 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/9df967d083a8470f98c255932b2c6524 2024-12-05T11:59:17,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/36ee47ed0fa84536a89c53154811270d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/36ee47ed0fa84536a89c53154811270d 2024-12-05T11:59:17,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/ae25df7e441a436aa57f96a53a247e45 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/ae25df7e441a436aa57f96a53a247e45 2024-12-05T11:59:17,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/1cafb44330104195b2bba9e8e6180a0f to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/1cafb44330104195b2bba9e8e6180a0f 2024-12-05T11:59:17,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/d07520c664fb4e36a1ae0e4a5630ce97 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/d07520c664fb4e36a1ae0e4a5630ce97 2024-12-05T11:59:17,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3da0aca2147040a79ed312a3673ff8ff to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3da0aca2147040a79ed312a3673ff8ff 2024-12-05T11:59:17,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/7bd9f5f02aae48dd86767b766398fd3e to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/7bd9f5f02aae48dd86767b766398fd3e 2024-12-05T11:59:17,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/eb0f6e1762a74ef2a7b3045230be2e77 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/eb0f6e1762a74ef2a7b3045230be2e77 2024-12-05T11:59:17,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/8e9f78bd87174d798e38158197900f4a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/8e9f78bd87174d798e38158197900f4a 2024-12-05T11:59:17,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/652d6342e4c04fed845c43d4a5778969 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/652d6342e4c04fed845c43d4a5778969 2024-12-05T11:59:17,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/39263e133f184d66b5d124867e2f7283 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/39263e133f184d66b5d124867e2f7283 2024-12-05T11:59:17,731 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/recovered.edits/401.seqid, newMaxSeqId=401, maxSeqId=4 2024-12-05T11:59:17,732 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. 2024-12-05T11:59:17,732 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for 990ce2fe1eb6f27987cd2f0b966a4627: Waiting for close lock at 1733399956435Running coprocessor pre-close hooks at 1733399956435Disabling compacts and flushes for region at 1733399956435Disabling writes for close at 1733399957117 (+682 ms)Obtaining lock to block concurrent updates at 1733399957117Preparing flush snapshotting stores in 990ce2fe1eb6f27987cd2f0b966a4627 at 1733399957117Finished memstore snapshotting TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1733399957117Flushing stores of TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627. at 1733399957118 (+1 ms)Flushing 990ce2fe1eb6f27987cd2f0b966a4627/A: creating writer at 1733399957118Flushing mob file 990ce2fe1eb6f27987cd2f0b966a4627/A: appending metadata at 1733399957122 (+4 ms)Flushing mob file 990ce2fe1eb6f27987cd2f0b966a4627/A: closing flushed file at 1733399957122Flushing 990ce2fe1eb6f27987cd2f0b966a4627/A: appending metadata at 1733399957129 (+7 ms)Flushing 990ce2fe1eb6f27987cd2f0b966a4627/A: closing flushed file at 1733399957129Flushing 990ce2fe1eb6f27987cd2f0b966a4627/B: creating writer at 1733399957135 (+6 ms)Flushing 990ce2fe1eb6f27987cd2f0b966a4627/B: appending metadata at 1733399957137 (+2 ms)Flushing 990ce2fe1eb6f27987cd2f0b966a4627/B: closing flushed file at 1733399957137Flushing 990ce2fe1eb6f27987cd2f0b966a4627/C: creating writer at 1733399957545 (+408 ms)Flushing 990ce2fe1eb6f27987cd2f0b966a4627/C: appending metadata at 1733399957547 (+2 ms)Flushing 990ce2fe1eb6f27987cd2f0b966a4627/C: closing flushed file at 1733399957548 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@493e3521: reopening flushed file at 1733399957584 (+36 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59ab5cf9: reopening flushed file at 1733399957590 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f1d826f: reopening flushed file at 1733399957597 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 990ce2fe1eb6f27987cd2f0b966a4627 in 486ms, sequenceid=398, compaction requested=true at 1733399957603 (+6 ms)Writing region close event to WAL at 1733399957725 (+122 ms)Running coprocessor post-close hooks at 1733399957732 (+7 ms)Closed at 1733399957732 2024-12-05T11:59:17,734 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed 990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:17,734 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=990ce2fe1eb6f27987cd2f0b966a4627, regionState=CLOSED 2024-12-05T11:59:17,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 because future has completed 2024-12-05T11:59:17,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-12-05T11:59:17,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure 990ce2fe1eb6f27987cd2f0b966a4627, server=80666a11a09f,40121,1733399769572 in 1.4540 sec 2024-12-05T11:59:17,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-12-05T11:59:17,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=990ce2fe1eb6f27987cd2f0b966a4627, UNASSIGN in 1.4600 sec 2024-12-05T11:59:17,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-05T11:59:17,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4670 sec 2024-12-05T11:59:17,746 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733399957745"}]},"ts":"1733399957745"} 2024-12-05T11:59:17,747 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T11:59:17,747 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T11:59:17,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4780 sec 2024-12-05T11:59:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-05T11:59:18,418 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:59:18,418 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T11:59:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:59:18,420 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:59:18,420 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:59:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-05T11:59:18,422 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,424 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C, FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/recovered.edits] 2024-12-05T11:59:18,426 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/056d1733be074290bc5cba45dcc6cee7 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/056d1733be074290bc5cba45dcc6cee7 2024-12-05T11:59:18,426 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0d7ac4a441404321b3cab5019c578298 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0d7ac4a441404321b3cab5019c578298 2024-12-05T11:59:18,427 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0f499cec035b49cba54d0b83a030e3bd to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/0f499cec035b49cba54d0b83a030e3bd 2024-12-05T11:59:18,428 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2050500d7ff440e78e18e1257f90a3b9 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2050500d7ff440e78e18e1257f90a3b9 2024-12-05T11:59:18,429 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2e2b47daafc44442a4b2e94cce9d2fe1 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/2e2b47daafc44442a4b2e94cce9d2fe1 2024-12-05T11:59:18,429 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/655f8fcbc9014eadb2a993040b06c8d7 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/A/655f8fcbc9014eadb2a993040b06c8d7 2024-12-05T11:59:18,431 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/503c9ee2f4c84ac98d7f75825d9f736d to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/503c9ee2f4c84ac98d7f75825d9f736d 2024-12-05T11:59:18,432 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/a0eca54303cc47ff9d15bfa5f2c3b9bb to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/a0eca54303cc47ff9d15bfa5f2c3b9bb 2024-12-05T11:59:18,433 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/aecd18fd3dd946e8bdc1fdd3df017f4a to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/aecd18fd3dd946e8bdc1fdd3df017f4a 2024-12-05T11:59:18,434 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b60baf65f9594815bfa2ada61e60531c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/b60baf65f9594815bfa2ada61e60531c 2024-12-05T11:59:18,435 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/d9b5f13dbe8746cd9b95362f06b77bd2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/d9b5f13dbe8746cd9b95362f06b77bd2 2024-12-05T11:59:18,436 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/e3038c7c56aa4c28bac33110655a8550 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/B/e3038c7c56aa4c28bac33110655a8550 2024-12-05T11:59:18,437 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/1422aaf052e64eb5bccede919d5a6eb2 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/1422aaf052e64eb5bccede919d5a6eb2 2024-12-05T11:59:18,438 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/27295fe7f0e145fa9d025d39200cd5bf to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/27295fe7f0e145fa9d025d39200cd5bf 2024-12-05T11:59:18,438 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3c52969b0d7d4d35b5be1ab3315783e3 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/3c52969b0d7d4d35b5be1ab3315783e3 2024-12-05T11:59:18,439 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/9cec692a2ebf4c3390846a7b4807a0a4 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/9cec692a2ebf4c3390846a7b4807a0a4 2024-12-05T11:59:18,440 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/a94fb3e709454359a1c87e2958486b0c to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/a94fb3e709454359a1c87e2958486b0c 2024-12-05T11:59:18,440 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/e670852710da4486b926afd7f20a04ca to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/C/e670852710da4486b926afd7f20a04ca 2024-12-05T11:59:18,443 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/recovered.edits/401.seqid to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627/recovered.edits/401.seqid 2024-12-05T11:59:18,443 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/default/TestAcidGuarantees/990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,443 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T11:59:18,444 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T11:59:18,444 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T11:59:18,446 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205113676bffe0948f89f50bd3e47fbc3f2_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205113676bffe0948f89f50bd3e47fbc3f2_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,447 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205131047b628a14f7982f782fd8d590dac_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205131047b628a14f7982f782fd8d590dac_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,448 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052ffce32271d24a69afb456a4388e020e_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052ffce32271d24a69afb456a4388e020e_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,449 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120536ae734c68934827a079ca50b4ebbd87_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120536ae734c68934827a079ca50b4ebbd87_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,450 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055ce66a940c6f49c984785c8669382bd7_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055ce66a940c6f49c984785c8669382bd7_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,451 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056108337036c74ee9835192e35ec1c1ec_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056108337036c74ee9835192e35ec1c1ec_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,452 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056957b2eb7b9a44d2a55afce462934b08_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056957b2eb7b9a44d2a55afce462934b08_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,452 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056c2101c45ce147c2a67e7d9da68555ee_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056c2101c45ce147c2a67e7d9da68555ee_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,453 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057532d62d62164c3486fbe07e80e2106c_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057532d62d62164c3486fbe07e80e2106c_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,454 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205877643b8926d4519bf7a1e37052d3559_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205877643b8926d4519bf7a1e37052d3559_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,455 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059623b6f582f24263bc0fa82700017678_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059623b6f582f24263bc0fa82700017678_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,455 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a0f3970cdda94deda0e195f80cd51c7f_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a0f3970cdda94deda0e195f80cd51c7f_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,456 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ac9913ef8a82443e87c721e57c8a2d4a_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ac9913ef8a82443e87c721e57c8a2d4a_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,457 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b703bc27ceba40c1abfb1a708ce17451_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b703bc27ceba40c1abfb1a708ce17451_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,457 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b90caa97ceea4ec18574cfe804d4bbde_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b90caa97ceea4ec18574cfe804d4bbde_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,459 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ba0389c0aced44ffaf72df7a85d47e9a_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ba0389c0aced44ffaf72df7a85d47e9a_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,460 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c0918d906db14f4e91194ba2849d5303_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c0918d906db14f4e91194ba2849d5303_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,461 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e653ecc402fe416fbe2ce5c4178a9ff9_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e653ecc402fe416fbe2ce5c4178a9ff9_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,462 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205eccc2e4cee1e49059ae8dfe9aeedb93b_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205eccc2e4cee1e49059ae8dfe9aeedb93b_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,463 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f48f36476f5d417c9b3f480639ac4bbb_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f48f36476f5d417c9b3f480639ac4bbb_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,463 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f89b11549d0b428da92d85515c51e4da_990ce2fe1eb6f27987cd2f0b966a4627 to hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f89b11549d0b428da92d85515c51e4da_990ce2fe1eb6f27987cd2f0b966a4627 2024-12-05T11:59:18,464 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T11:59:18,465 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:59:18,468 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T11:59:18,469 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T11:59:18,470 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:59:18,470 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T11:59:18,470 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733399958470"}]},"ts":"9223372036854775807"} 2024-12-05T11:59:18,472 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T11:59:18,472 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 990ce2fe1eb6f27987cd2f0b966a4627, NAME => 'TestAcidGuarantees,,1733399932408.990ce2fe1eb6f27987cd2f0b966a4627.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T11:59:18,472 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T11:59:18,472 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733399958472"}]},"ts":"9223372036854775807"} 2024-12-05T11:59:18,474 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T11:59:18,475 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:59:18,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-12-05T11:59:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-05T11:59:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T11:59:18,529 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:59:18,537 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239 (was 239), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=590 (was 562) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6717 (was 6829) 2024-12-05T11:59:18,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T11:59:18,537 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:59:18,538 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:59:18,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:18,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:18,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:59:18,538 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T11:59:18,538 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=611241949, stopped=false 2024-12-05T11:59:18,538 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=80666a11a09f,33741,1733399768894 2024-12-05T11:59:18,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T11:59:18,585 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T11:59:18,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:59:18,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T11:59:18,585 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:59:18,585 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:59:18,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:59:18,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:18,586 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '80666a11a09f,40121,1733399769572' ***** 2024-12-05T11:59:18,586 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T11:59:18,586 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T11:59:18,587 INFO [RS:0;80666a11a09f:40121 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T11:59:18,587 INFO [RS:0;80666a11a09f:40121 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T11:59:18,587 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T11:59:18,587 INFO [RS:0;80666a11a09f:40121 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T11:59:18,587 INFO [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(959): stopping server 80666a11a09f,40121,1733399769572 2024-12-05T11:59:18,587 INFO [RS:0;80666a11a09f:40121 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T11:59:18,587 INFO [RS:0;80666a11a09f:40121 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;80666a11a09f:40121. 2024-12-05T11:59:18,587 DEBUG [RS:0;80666a11a09f:40121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:59:18,587 DEBUG [RS:0;80666a11a09f:40121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:18,589 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T11:59:18,590 INFO [RS:0;80666a11a09f:40121 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T11:59:18,590 INFO [RS:0;80666a11a09f:40121 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T11:59:18,590 INFO [RS:0;80666a11a09f:40121 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T11:59:18,590 INFO [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T11:59:18,591 INFO [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-05T11:59:18,591 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-05T11:59:18,591 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:59:18,591 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T11:59:18,591 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T11:59:18,591 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T11:59:18,591 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T11:59:18,591 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T11:59:18,591 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-05T11:59:18,614 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/.tmp/info/1bb58ff4a1fe42b2910f78503c629d4f is 95, key is TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150./info:/1733399800966/DeleteFamily/seqid=0 2024-12-05T11:59:18,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742465_1641 (size=6514) 2024-12-05T11:59:18,685 INFO [regionserver/80666a11a09f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T11:59:18,791 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:59:18,991 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:59:19,020 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/.tmp/info/1bb58ff4a1fe42b2910f78503c629d4f 2024-12-05T11:59:19,041 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/.tmp/ns/d292b084cd834079a5340afd5300f155 is 93, key is TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150./ns:/1733399800934/DeleteFamily/seqid=0 2024-12-05T11:59:19,042 INFO [regionserver/80666a11a09f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T11:59:19,043 INFO [regionserver/80666a11a09f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T11:59:19,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742466_1642 (size=5906) 2024-12-05T11:59:19,192 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:59:19,211 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T11:59:19,392 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:59:19,445 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/.tmp/ns/d292b084cd834079a5340afd5300f155 2024-12-05T11:59:19,463 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/.tmp/rep_barrier/ec0e04657a6443328192ebac56695737 is 102, key is TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150./rep_barrier:/1733399800934/DeleteFamily/seqid=0 2024-12-05T11:59:19,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742467_1643 (size=6025) 2024-12-05T11:59:19,592 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:59:19,792 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:59:19,866 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/.tmp/rep_barrier/ec0e04657a6443328192ebac56695737 2024-12-05T11:59:19,886 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/.tmp/table/224d0f535f094bbca43e7ea9336781bc is 96, key is TestAcidGuarantees,,1733399773687.41e385893d26f932b2fcba15c0b7f150./table:/1733399800934/DeleteFamily/seqid=0 2024-12-05T11:59:19,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742468_1644 (size=5936) 2024-12-05T11:59:19,992 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:59:20,193 DEBUG [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:59:20,290 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/.tmp/table/224d0f535f094bbca43e7ea9336781bc 2024-12-05T11:59:20,293 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/.tmp/info/1bb58ff4a1fe42b2910f78503c629d4f as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/info/1bb58ff4a1fe42b2910f78503c629d4f 2024-12-05T11:59:20,296 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/info/1bb58ff4a1fe42b2910f78503c629d4f, entries=12, sequenceid=90, filesize=6.4 K 2024-12-05T11:59:20,297 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/.tmp/ns/d292b084cd834079a5340afd5300f155 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/ns/d292b084cd834079a5340afd5300f155 2024-12-05T11:59:20,300 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/ns/d292b084cd834079a5340afd5300f155, entries=8, sequenceid=90, filesize=5.8 K 2024-12-05T11:59:20,300 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/.tmp/rep_barrier/ec0e04657a6443328192ebac56695737 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/rep_barrier/ec0e04657a6443328192ebac56695737 2024-12-05T11:59:20,304 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/rep_barrier/ec0e04657a6443328192ebac56695737, entries=6, sequenceid=90, filesize=5.9 K 2024-12-05T11:59:20,304 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/.tmp/table/224d0f535f094bbca43e7ea9336781bc as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/table/224d0f535f094bbca43e7ea9336781bc 2024-12-05T11:59:20,307 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/table/224d0f535f094bbca43e7ea9336781bc, entries=7, sequenceid=90, filesize=5.8 K 2024-12-05T11:59:20,308 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1717ms, sequenceid=90, compaction requested=false 2024-12-05T11:59:20,317 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-05T11:59:20,318 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T11:59:20,318 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T11:59:20,318 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733399958591Running coprocessor pre-close hooks at 1733399958591Disabling compacts and flushes for region at 1733399958591Disabling writes for close at 1733399958591Obtaining lock to block concurrent updates at 1733399958591Preparing flush snapshotting stores in 1588230740 at 1733399958591Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733399958592 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733399958592Flushing 1588230740/info: creating writer at 1733399958592Flushing 1588230740/info: appending metadata at 1733399958613 (+21 ms)Flushing 1588230740/info: closing flushed file at 1733399958613Flushing 1588230740/ns: creating writer at 1733399959024 (+411 ms)Flushing 1588230740/ns: appending metadata at 1733399959041 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733399959041Flushing 1588230740/rep_barrier: creating writer at 1733399959448 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733399959462 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733399959462Flushing 1588230740/table: creating writer at 1733399959869 (+407 ms)Flushing 1588230740/table: appending metadata at 1733399959886 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733399959886Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4409515: reopening flushed file at 1733399960293 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e0a6693: reopening flushed file at 1733399960296 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74312cda: reopening flushed file at 1733399960300 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b5c9335: reopening flushed file at 1733399960304 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1717ms, sequenceid=90, compaction requested=false at 1733399960308 (+4 ms)Writing region close event to WAL at 1733399960315 (+7 ms)Running coprocessor post-close hooks at 1733399960318 (+3 ms)Closed at 1733399960318 2024-12-05T11:59:20,318 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T11:59:20,393 INFO [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(976): stopping server 80666a11a09f,40121,1733399769572; all regions closed. 2024-12-05T11:59:20,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741834_1010 (size=25344) 2024-12-05T11:59:20,399 DEBUG [RS:0;80666a11a09f:40121 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/oldWALs 2024-12-05T11:59:20,399 INFO [RS:0;80666a11a09f:40121 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80666a11a09f%2C40121%2C1733399769572.meta:.meta(num 1733399773272) 2024-12-05T11:59:20,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741832_1008 (size=14776395) 2024-12-05T11:59:20,403 DEBUG [RS:0;80666a11a09f:40121 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/oldWALs 2024-12-05T11:59:20,403 INFO [RS:0;80666a11a09f:40121 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80666a11a09f%2C40121%2C1733399769572:(num 1733399772193) 2024-12-05T11:59:20,403 DEBUG [RS:0;80666a11a09f:40121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:59:20,403 INFO [RS:0;80666a11a09f:40121 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T11:59:20,403 INFO [RS:0;80666a11a09f:40121 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T11:59:20,403 INFO [RS:0;80666a11a09f:40121 {}] hbase.ChoreService(370): Chore service for: regionserver/80666a11a09f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T11:59:20,403 INFO [RS:0;80666a11a09f:40121 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T11:59:20,403 INFO [regionserver/80666a11a09f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T11:59:20,404 INFO [RS:0;80666a11a09f:40121 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40121 2024-12-05T11:59:20,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T11:59:20,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80666a11a09f,40121,1733399769572 2024-12-05T11:59:20,443 INFO [RS:0;80666a11a09f:40121 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T11:59:20,444 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80666a11a09f,40121,1733399769572] 2024-12-05T11:59:20,454 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/80666a11a09f,40121,1733399769572 already deleted, retry=false 2024-12-05T11:59:20,454 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 80666a11a09f,40121,1733399769572 expired; onlineServers=0 2024-12-05T11:59:20,454 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '80666a11a09f,33741,1733399768894' ***** 2024-12-05T11:59:20,455 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T11:59:20,455 INFO [M:0;80666a11a09f:33741 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T11:59:20,455 INFO [M:0;80666a11a09f:33741 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T11:59:20,455 DEBUG [M:0;80666a11a09f:33741 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T11:59:20,455 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T11:59:20,455 DEBUG [M:0;80666a11a09f:33741 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T11:59:20,455 DEBUG [master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.small.0-1733399771945 {}] cleaner.HFileCleaner(306): Exit Thread[master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.small.0-1733399771945,5,FailOnTimeoutGroup] 2024-12-05T11:59:20,455 INFO [M:0;80666a11a09f:33741 {}] hbase.ChoreService(370): Chore service for: master/80666a11a09f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T11:59:20,455 INFO [M:0;80666a11a09f:33741 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T11:59:20,455 DEBUG [M:0;80666a11a09f:33741 {}] master.HMaster(1795): Stopping service threads 2024-12-05T11:59:20,455 INFO [M:0;80666a11a09f:33741 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T11:59:20,455 INFO [M:0;80666a11a09f:33741 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T11:59:20,456 ERROR [M:0;80666a11a09f:33741 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:40895 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:40895,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-05T11:59:20,456 INFO [M:0;80666a11a09f:33741 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T11:59:20,456 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T11:59:20,456 DEBUG [master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.large.0-1733399771941 {}] cleaner.HFileCleaner(306): Exit Thread[master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.large.0-1733399771941,5,FailOnTimeoutGroup] 2024-12-05T11:59:20,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T11:59:20,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:59:20,462 DEBUG [M:0;80666a11a09f:33741 {}] zookeeper.ZKUtil(347): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T11:59:20,463 WARN [M:0;80666a11a09f:33741 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T11:59:20,463 INFO [M:0;80666a11a09f:33741 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/.lastflushedseqids 2024-12-05T11:59:20,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742469_1645 (size=72) 2024-12-05T11:59:20,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T11:59:20,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40121-0x101a6a30e880001, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T11:59:20,555 INFO [RS:0;80666a11a09f:40121 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T11:59:20,555 INFO [RS:0;80666a11a09f:40121 {}] regionserver.HRegionServer(1031): Exiting; stopping=80666a11a09f,40121,1733399769572; zookeeper connection closed. 2024-12-05T11:59:20,555 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a1f568b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a1f568b 2024-12-05T11:59:20,556 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T11:59:20,872 INFO [M:0;80666a11a09f:33741 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T11:59:20,872 INFO [M:0;80666a11a09f:33741 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T11:59:20,872 DEBUG [M:0;80666a11a09f:33741 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T11:59:20,872 INFO [M:0;80666a11a09f:33741 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:59:20,872 DEBUG [M:0;80666a11a09f:33741 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:59:20,872 DEBUG [M:0;80666a11a09f:33741 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T11:59:20,872 DEBUG [M:0;80666a11a09f:33741 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:59:20,872 INFO [M:0;80666a11a09f:33741 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=762.65 KB heapSize=938.13 KB 2024-12-05T11:59:20,887 DEBUG [M:0;80666a11a09f:33741 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c681be6341d24757997a018b17e83e41 is 82, key is hbase:meta,,1/info:regioninfo/1733399773349/Put/seqid=0 2024-12-05T11:59:20,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742470_1646 (size=5672) 2024-12-05T11:59:21,295 INFO [M:0;80666a11a09f:33741 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2163 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c681be6341d24757997a018b17e83e41 2024-12-05T11:59:21,319 DEBUG [M:0;80666a11a09f:33741 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b798f9db09904ebcabaa457cc894636e is 2275, key is \x00\x00\x00\x00\x00\x00\x00Z/proc:d/1733399874580/Put/seqid=0 2024-12-05T11:59:21,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742471_1647 (size=44822) 2024-12-05T11:59:21,723 INFO [M:0;80666a11a09f:33741 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=762.10 KB at sequenceid=2163 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b798f9db09904ebcabaa457cc894636e 2024-12-05T11:59:21,728 INFO [M:0;80666a11a09f:33741 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b798f9db09904ebcabaa457cc894636e 2024-12-05T11:59:21,744 DEBUG [M:0;80666a11a09f:33741 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dd83683678aa4339a8be7e86f987bde9 is 69, key is 80666a11a09f,40121,1733399769572/rs:state/1733399771963/Put/seqid=0 2024-12-05T11:59:21,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073742472_1648 (size=5156) 2024-12-05T11:59:22,148 INFO [M:0;80666a11a09f:33741 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2163 (bloomFilter=true), to=hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dd83683678aa4339a8be7e86f987bde9 2024-12-05T11:59:22,151 DEBUG [M:0;80666a11a09f:33741 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c681be6341d24757997a018b17e83e41 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c681be6341d24757997a018b17e83e41 2024-12-05T11:59:22,153 INFO [M:0;80666a11a09f:33741 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c681be6341d24757997a018b17e83e41, entries=8, sequenceid=2163, filesize=5.5 K 2024-12-05T11:59:22,154 DEBUG [M:0;80666a11a09f:33741 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b798f9db09904ebcabaa457cc894636e as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b798f9db09904ebcabaa457cc894636e 2024-12-05T11:59:22,156 INFO [M:0;80666a11a09f:33741 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b798f9db09904ebcabaa457cc894636e 2024-12-05T11:59:22,156 INFO [M:0;80666a11a09f:33741 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b798f9db09904ebcabaa457cc894636e, entries=174, sequenceid=2163, filesize=43.8 K 2024-12-05T11:59:22,157 DEBUG [M:0;80666a11a09f:33741 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dd83683678aa4339a8be7e86f987bde9 as hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dd83683678aa4339a8be7e86f987bde9 2024-12-05T11:59:22,159 INFO [M:0;80666a11a09f:33741 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40895/user/jenkins/test-data/772ab5be-5e45-96fa-1a7b-f5ea265f9d8c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dd83683678aa4339a8be7e86f987bde9, entries=1, sequenceid=2163, filesize=5.0 K 2024-12-05T11:59:22,160 INFO [M:0;80666a11a09f:33741 {}] regionserver.HRegion(3140): Finished flush of dataSize ~762.65 KB/780957, heapSize ~937.83 KB/960336, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1288ms, sequenceid=2163, compaction requested=false 2024-12-05T11:59:22,161 INFO [M:0;80666a11a09f:33741 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:59:22,161 DEBUG [M:0;80666a11a09f:33741 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733399960872Disabling compacts and flushes for region at 1733399960872Disabling writes for close at 1733399960872Obtaining lock to block concurrent updates at 1733399960872Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733399960872Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=780957, getHeapSize=960576, getOffHeapSize=0, getCellsCount=2362 at 1733399960873 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733399960873Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733399960873Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733399960887 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733399960887Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733399961300 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733399961319 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733399961319Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733399961728 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733399961744 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733399961744Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e7cdf87: reopening flushed file at 1733399962150 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@659e9960: reopening flushed file at 1733399962153 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54973986: reopening flushed file at 1733399962156 (+3 ms)Finished flush of dataSize ~762.65 KB/780957, heapSize ~937.83 KB/960336, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1288ms, sequenceid=2163, compaction requested=false at 1733399962160 (+4 ms)Writing region close event to WAL at 1733399962161 (+1 ms)Closed at 1733399962161 2024-12-05T11:59:22,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36633 is added to blk_1073741830_1006 (size=922767) 2024-12-05T11:59:22,164 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T11:59:22,164 INFO [M:0;80666a11a09f:33741 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T11:59:22,164 INFO [M:0;80666a11a09f:33741 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33741 2024-12-05T11:59:22,164 INFO [M:0;80666a11a09f:33741 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T11:59:22,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T11:59:22,311 INFO [M:0;80666a11a09f:33741 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T11:59:22,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x101a6a30e880000, quorum=127.0.0.1:58051, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T11:59:22,349 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T11:59:22,354 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T11:59:22,354 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T11:59:22,354 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T11:59:22,355 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/hadoop.log.dir/,STOPPED} 2024-12-05T11:59:22,357 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T11:59:22,357 WARN [BP-897299292-172.17.0.2-1733399765732 heartbeating to localhost/127.0.0.1:40895 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T11:59:22,357 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T11:59:22,357 WARN [BP-897299292-172.17.0.2-1733399765732 heartbeating to localhost/127.0.0.1:40895 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-897299292-172.17.0.2-1733399765732 (Datanode Uuid cf0fba41-3359-4e83-a698-6fd216eccf92) service to localhost/127.0.0.1:40895 2024-12-05T11:59:22,359 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/cluster_f9008675-76bd-e1e2-e855-619669955c54/data/data1/current/BP-897299292-172.17.0.2-1733399765732 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T11:59:22,359 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/cluster_f9008675-76bd-e1e2-e855-619669955c54/data/data2/current/BP-897299292-172.17.0.2-1733399765732 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T11:59:22,359 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T11:59:22,365 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T11:59:22,366 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T11:59:22,366 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T11:59:22,366 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T11:59:22,366 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d31f3c0f-04ac-7c9b-8596-f368b62dbe51/hadoop.log.dir/,STOPPED} 2024-12-05T11:59:22,385 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T11:59:22,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down